serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-26 00:13:47 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

switch to slog.Logger for logging, for easier reuse of packages by external software

Browse source 
we don't want external software to include internal details like mlog.
slog.Logger is/will be the standard.

we still have mlog for its helper functions, and its handler that logs in
concise logfmt used by mox.

packages that are not meant for reuse still pass around mlog.Log for
convenience.

we use golang.org/x/exp/slog because we also support the previous Go toolchain
version. with the next Go release, we'll switch to the builtin slog.
This commit is contained in:
Mechiel Lukkien 2023-12-05 13:35:58 +01:00
parent 56b2a9d980
commit 5b20cba50a
No known key found for this signature in database
150 changed files with 5176 additions and 1898 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
autotls/autotls.go
View file

@ -28,9 +28,11 @@ import (
"sync" "sync"
"time" "time"
"golang.org/x/crypto/acme"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
"golang.org/x/crypto/acme"
"github.com/mjl-/autocert" "github.com/mjl-/autocert"
@ -39,8 +41,6 @@ import (
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/moxvar"
) )
var xlog = mlog.New("autotls")
var ( var (
metricCertput = promauto.NewCounter( metricCertput = promauto.NewCounter(
prometheus.CounterOpts{ prometheus.CounterOpts{
@ -148,7 +148,7 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h
} }
loggingGetCertificate := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { loggingGetCertificate := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
log := xlog.WithContext(hello.Context()) log := mlog.New("autotls", nil).WithContext(hello.Context())
// Handle missing SNI to prevent logging an error below. // Handle missing SNI to prevent logging an error below.
// At startup, during config initialization, we already adjust the tls config to // At startup, during config initialization, we already adjust the tls config to
@ -156,16 +156,16 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h
// common for SMTP STARTTLS connections, which often do not care about the // common for SMTP STARTTLS connections, which often do not care about the
// verification of the certificate. // verification of the certificate.
if hello.ServerName == "" { if hello.ServerName == "" {
log.Debug("tls request without sni servername, rejecting", mlog.Field("localaddr", hello.Conn.LocalAddr()), mlog.Field("supportedprotos", hello.SupportedProtos)) log.Debug("tls request without sni servername, rejecting", slog.Any("localaddr", hello.Conn.LocalAddr()), slog.Any("supportedprotos", hello.SupportedProtos))
return nil, fmt.Errorf("sni server name required") return nil, fmt.Errorf("sni server name required")
} }
cert, err := m.GetCertificate(hello) cert, err := m.GetCertificate(hello)
if err != nil { if err != nil {
if errors.Is(err, errHostNotAllowed) { if errors.Is(err, errHostNotAllowed) {
log.Debugx("requesting certificate", err, mlog.Field("host", hello.ServerName)) log.Debugx("requesting certificate", err, slog.String("host", hello.ServerName))
} else { } else {
log.Errorx("requesting certificate", err, mlog.Field("host", hello.ServerName)) log.Errorx("requesting certificate", err, slog.String("host", hello.ServerName))
} }
} }
return cert, err return cert, err
@ -194,7 +194,7 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h
// are fully served by publicIPs (only if non-empty and there is no unspecified // are fully served by publicIPs (only if non-empty and there is no unspecified
// address in the list). If no, log an error with a warning that ACME validation // address in the list). If no, log an error with a warning that ACME validation
// may fail. // may fail.
func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.Domain]struct{}, publicIPs []string, checkHosts bool) { func (m *Manager) SetAllowedHostnames(log mlog.Log, resolver dns.Resolver, hostnames map[dns.Domain]struct{}, publicIPs []string, checkHosts bool) {
m.Lock() m.Lock()
defer m.Unlock() defer m.Unlock()
@ -207,7 +207,7 @@ func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.D
return l[i].Name() < l[j].Name() return l[i].Name() < l[j].Name()
}) })
xlog.Debug("autotls setting allowed hostnames", mlog.Field("hostnames", l), mlog.Field("publicips", publicIPs)) log.Debug("autotls setting allowed hostnames", slog.Any("hostnames", l), slog.Any("publicips", publicIPs))
var added []dns.Domain var added []dns.Domain
for h := range hostnames { for h := range hostnames {
if _, ok := m.hosts[h]; !ok { if _, ok := m.hosts[h]; !ok {
@ -231,16 +231,16 @@ func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.D
publicIPstrs[ip] = struct{}{} publicIPstrs[ip] = struct{}{}
} }
xlog.Debug("checking ips of hosts configured for acme tls cert validation") log.Debug("checking ips of hosts configured for acme tls cert validation")
for _, h := range added { for _, h := range added {
ips, _, err := resolver.LookupIP(ctx, "ip", h.ASCII+".") ips, _, err := resolver.LookupIP(ctx, "ip", h.ASCII+".")
if err != nil { if err != nil {
xlog.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, mlog.Field("host", h)) log.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, slog.Any("host", h))
continue continue
} }
for _, ip := range ips { for _, ip := range ips {
if _, ok := publicIPstrs[ip.String()]; !ok { if _, ok := publicIPstrs[ip.String()]; !ok {
xlog.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on", mlog.Field("hostname", h), mlog.Field("listenedips", publicIPs), mlog.Field("hostips", ips), mlog.Field("missingip", ip)) log.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on", slog.Any("hostname", h), slog.Any("listenedips", publicIPs), slog.Any("hostips", ips), slog.Any("missingip", ip))
} }
} }
} }
@ -266,9 +266,9 @@ var errHostNotAllowed = errors.New("autotls: host not in allowlist")
// present. Only hosts added with SetAllowedHostnames are allowed. During shutdown, // present. Only hosts added with SetAllowedHostnames are allowed. During shutdown,
// no new connections are allowed. // no new connections are allowed.
func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) { func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("autotls", nil).WithContext(ctx)
defer func() { defer func() {
log.WithContext(ctx).Debugx("autotls hostpolicy result", rerr, mlog.Field("host", host)) log.Debugx("autotls hostpolicy result", rerr, slog.String("host", host))
}() }()
// Don't request new TLS certs when we are shutting down. // Don't request new TLS certs when we are shutting down.
@ -300,46 +300,46 @@ func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) {
type dirCache autocert.DirCache type dirCache autocert.DirCache
func (d dirCache) Delete(ctx context.Context, name string) (rerr error) { func (d dirCache) Delete(ctx context.Context, name string) (rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("autotls", nil).WithContext(ctx)
defer func() { defer func() {
log.Debugx("dircache delete result", rerr, mlog.Field("name", name)) log.Debugx("dircache delete result", rerr, slog.String("name", name))
}() }()
err := autocert.DirCache(d).Delete(ctx, name) err := autocert.DirCache(d).Delete(ctx, name)
if err != nil { if err != nil {
log.Errorx("deleting cert from dir cache", err, mlog.Field("name", name)) log.Errorx("deleting cert from dir cache", err, slog.String("name", name))
} else if !strings.HasSuffix(name, "+token") { } else if !strings.HasSuffix(name, "+token") {
log.Info("autotls cert delete", mlog.Field("name", name)) log.Info("autotls cert delete", slog.String("name", name))
} }
return err return err
} }
func (d dirCache) Get(ctx context.Context, name string) (rbuf []byte, rerr error) { func (d dirCache) Get(ctx context.Context, name string) (rbuf []byte, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("autotls", nil).WithContext(ctx)
defer func() { defer func() {
log.Debugx("dircache get result", rerr, mlog.Field("name", name)) log.Debugx("dircache get result", rerr, slog.String("name", name))
}() }()
buf, err := autocert.DirCache(d).Get(ctx, name) buf, err := autocert.DirCache(d).Get(ctx, name)
if err != nil && errors.Is(err, autocert.ErrCacheMiss) { if err != nil && errors.Is(err, autocert.ErrCacheMiss) {
log.Infox("getting cert from dir cache", err, mlog.Field("name", name)) log.Infox("getting cert from dir cache", err, slog.String("name", name))
} else if err != nil { } else if err != nil {
log.Errorx("getting cert from dir cache", err, mlog.Field("name", name)) log.Errorx("getting cert from dir cache", err, slog.String("name", name))
} else if !strings.HasSuffix(name, "+token") { } else if !strings.HasSuffix(name, "+token") {
log.Debug("autotls cert get", mlog.Field("name", name)) log.Debug("autotls cert get", slog.String("name", name))
} }
return buf, err return buf, err
} }
func (d dirCache) Put(ctx context.Context, name string, data []byte) (rerr error) { func (d dirCache) Put(ctx context.Context, name string, data []byte) (rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("autotls", nil).WithContext(ctx)
defer func() { defer func() {
log.Debugx("dircache put result", rerr, mlog.Field("name", name)) log.Debugx("dircache put result", rerr, slog.String("name", name))
}() }()
metricCertput.Inc() metricCertput.Inc()
err := autocert.DirCache(d).Put(ctx, name, data) err := autocert.DirCache(d).Put(ctx, name, data)
if err != nil { if err != nil {
log.Errorx("storing cert in dir cache", err, mlog.Field("name", name)) log.Errorx("storing cert in dir cache", err, slog.String("name", name))
} else if !strings.HasSuffix(name, "+token") { } else if !strings.HasSuffix(name, "+token") {
log.Info("autotls cert store", mlog.Field("name", name)) log.Info("autotls cert store", slog.String("name", name))
} }
return err return err
} }

6
autotls/autotls_test.go
View file

@ -12,9 +12,11 @@ import (
"github.com/mjl-/autocert" "github.com/mjl-/autocert"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
) )
func TestAutotls(t *testing.T) { func TestAutotls(t *testing.T) {
log := mlog.New("autotls", nil)
os.RemoveAll("../testdata/autotls") os.RemoveAll("../testdata/autotls")
os.MkdirAll("../testdata/autotls", 0770) os.MkdirAll("../testdata/autotls", 0770)
@ -34,7 +36,7 @@ func TestAutotls(t *testing.T) {
if err := m.HostPolicy(context.Background(), "mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) { if err := m.HostPolicy(context.Background(), "mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) {
t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err) t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err)
} }
m.SetAllowedHostnames(dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false) m.SetAllowedHostnames(log, dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false)
l = m.Hostnames() l = m.Hostnames()
if !reflect.DeepEqual(l, []dns.Domain{{ASCII: "mox.example"}}) { if !reflect.DeepEqual(l, []dns.Domain{{ASCII: "mox.example"}}) {
t.Fatalf("hostnames, got %v, expected single mox.example", l) t.Fatalf("hostnames, got %v, expected single mox.example", l)
@ -88,7 +90,7 @@ func TestAutotls(t *testing.T) {
t.Fatalf("private key changed after reload") t.Fatalf("private key changed after reload")
} }
m.shutdown = make(chan struct{}) m.shutdown = make(chan struct{})
m.SetAllowedHostnames(dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false) m.SetAllowedHostnames(log, dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false)
if err := m.HostPolicy(context.Background(), "mox.example"); err != nil { if err := m.HostPolicy(context.Background(), "mox.example"); err != nil {
t.Fatalf("hostpolicy, got err %v, expected no error", err) t.Fatalf("hostpolicy, got err %v, expected no error", err)
} }

111
backup.go
View file

@ -12,10 +12,11 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dmarcdb" "github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/mtastsdb" "github.com/mjl-/mox/mtastsdb"
@ -48,51 +49,51 @@ func backupctl(ctx context.Context, ctl *ctl) {
writer := ctl.writer() writer := ctl.writer()
// Format easily readable output for the user. // Format easily readable output for the user.
formatLog := func(prefix, text string, err error, fields ...mlog.Pair) []byte { formatLog := func(prefix, text string, err error, attrs ...slog.Attr) []byte {
var b bytes.Buffer var b bytes.Buffer
fmt.Fprint(&b, prefix) fmt.Fprint(&b, prefix)
fmt.Fprint(&b, text) fmt.Fprint(&b, text)
if err != nil { if err != nil {
fmt.Fprint(&b, ": "+err.Error()) fmt.Fprint(&b, ": "+err.Error())
} }
for _, f := range fields { for _, a := range attrs {
fmt.Fprintf(&b, "; %s=%v", f.Key, f.Value) fmt.Fprintf(&b, "; %s=%v", a.Key, a.Value)
} }
fmt.Fprint(&b, "\n") fmt.Fprint(&b, "\n")
return b.Bytes() return b.Bytes()
} }
// Log an error to both the mox service as the user running "mox backup". // Log an error to both the mox service as the user running "mox backup".
xlogx := func(prefix, text string, err error, fields ...mlog.Pair) { pkglogx := func(prefix, text string, err error, attrs ...slog.Attr) {
ctl.log.Errorx(text, err, fields...) ctl.log.Errorx(text, err, attrs...)
_, werr := writer.Write(formatLog(prefix, text, err, fields...)) _, werr := writer.Write(formatLog(prefix, text, err, attrs...))
ctl.xcheck(werr, "write to ctl") ctl.xcheck(werr, "write to ctl")
} }
// Log an error but don't mark backup as failed. // Log an error but don't mark backup as failed.
xwarnx := func(text string, err error, fields ...mlog.Pair) { xwarnx := func(text string, err error, attrs ...slog.Attr) {
xlogx("warning: ", text, err, fields...) pkglogx("warning: ", text, err, attrs...)
} }
// Log an error that causes the backup to be marked as failed. We typically // Log an error that causes the backup to be marked as failed. We typically
// continue processing though. // continue processing though.
xerrx := func(text string, err error, fields ...mlog.Pair) { xerrx := func(text string, err error, attrs ...slog.Attr) {
incomplete = true incomplete = true
xlogx("error: ", text, err, fields...) pkglogx("error: ", text, err, attrs...)
} }
// If verbose is enabled, log to the cli command. Always log as info level. // If verbose is enabled, log to the cli command. Always log as info level.
xvlog := func(text string, fields ...mlog.Pair) { xvlog := func(text string, attrs ...slog.Attr) {
ctl.log.Info(text, fields...) ctl.log.Info(text, attrs...)
if verbose { if verbose {
_, werr := writer.Write(formatLog("", text, nil, fields...)) _, werr := writer.Write(formatLog("", text, nil, attrs...))
ctl.xcheck(werr, "write to ctl") ctl.xcheck(werr, "write to ctl")
} }
} }
if _, err := os.Stat(dstDataDir); err == nil { if _, err := os.Stat(dstDataDir); err == nil {
xwarnx("destination data directory already exists", nil, mlog.Field("dir", dstDataDir)) xwarnx("destination data directory already exists", nil, slog.String("dir", dstDataDir))
} }
srcDataDir := filepath.Clean(mox.DataDirPath(".")) srcDataDir := filepath.Clean(mox.DataDirPath("."))
@ -119,7 +120,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
sf, err := os.Open(srcpath) sf, err := os.Open(srcpath)
if err != nil { if err != nil {
xerrx("open source file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xerrx("open source file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return return
} }
defer sf.Close() defer sf.Close()
@ -127,7 +128,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
ensureDestDir(dstpath) ensureDestDir(dstpath)
df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
if err != nil { if err != nil {
xerrx("creating destination file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xerrx("creating destination file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return return
} }
defer func() { defer func() {
@ -136,16 +137,16 @@ func backupctl(ctx context.Context, ctl *ctl) {
} }
}() }()
if _, err := io.Copy(df, sf); err != nil { if _, err := io.Copy(df, sf); err != nil {
xerrx("copying file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xerrx("copying file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return return
} }
err = df.Close() err = df.Close()
df = nil df = nil
if err != nil { if err != nil {
xerrx("closing destination file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xerrx("closing destination file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return return
} }
xvlog("backed up file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmFile))) xvlog("backed up file", slog.String("path", path), slog.Duration("duration", time.Since(tmFile)))
} }
// Back up the files in a directory (by copying). // Back up the files in a directory (by copying).
@ -155,7 +156,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
dstdir := filepath.Join(dstDataDir, dir) dstdir := filepath.Join(dstDataDir, dir)
err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error { err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
xerrx("walking file (not backed up)", err, mlog.Field("srcpath", srcpath)) xerrx("walking file (not backed up)", err, slog.String("srcpath", srcpath))
return nil return nil
} }
if d.IsDir() { if d.IsDir() {
@ -165,10 +166,10 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil return nil
}) })
if err != nil { if err != nil {
xerrx("copying directory (not backed up properly)", err, mlog.Field("srcdir", srcdir), mlog.Field("dstdir", dstdir), mlog.Field("duration", time.Since(tmDir))) xerrx("copying directory (not backed up properly)", err, slog.String("srcdir", srcdir), slog.String("dstdir", dstdir), slog.Duration("duration", time.Since(tmDir)))
return return
} }
xvlog("backed up directory", mlog.Field("dir", dir), mlog.Field("duration", time.Since(tmDir))) xvlog("backed up directory", slog.String("dir", dir), slog.Duration("duration", time.Since(tmDir)))
} }
// Backup a database by copying it in a readonly transaction. // Backup a database by copying it in a readonly transaction.
@ -177,7 +178,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
backupDB := func(db *bstore.DB, path string) (rerr error) { backupDB := func(db *bstore.DB, path string) (rerr error) {
defer func() { defer func() {
if rerr != nil { if rerr != nil {
xerrx("backing up database", rerr, mlog.Field("path", path)) xerrx("backing up database", rerr, slog.String("path", path))
} }
}() }()
@ -216,7 +217,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
if err != nil { if err != nil {
return fmt.Errorf("closing destination database after copy: %v", err) return fmt.Errorf("closing destination database after copy: %v", err)
} }
xvlog("backed up database file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmDB))) xvlog("backed up database file", slog.String("path", path), slog.Duration("duration", time.Since(tmDB)))
return nil return nil
} }
@ -231,7 +232,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
// No point in trying with regular copy, we would warn twice. // No point in trying with regular copy, we would warn twice.
return false, err return false, err
} else if !warnedHardlink { } else if !warnedHardlink {
xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
warnedHardlink = true warnedHardlink = true
} }
@ -269,7 +270,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
// Start making the backup. // Start making the backup.
tmStart := time.Now() tmStart := time.Now()
ctl.log.Print("making backup", mlog.Field("destdir", dstDataDir)) ctl.log.Print("making backup", slog.String("destdir", dstDataDir))
err := os.MkdirAll(dstDataDir, 0770) err := os.MkdirAll(dstDataDir, 0770)
if err != nil { if err != nil {
@ -299,14 +300,14 @@ func backupctl(ctx context.Context, ctl *ctl) {
tmQueue := time.Now() tmQueue := time.Now()
if err := backupDB(queue.DB, path); err != nil { if err := backupDB(queue.DB, path); err != nil {
xerrx("queue not backed up", err, mlog.Field("path", path), mlog.Field("duration", time.Since(tmQueue))) xerrx("queue not backed up", err, slog.String("path", path), slog.Duration("duration", time.Since(tmQueue)))
return return
} }
dstdbpath := filepath.Join(dstDataDir, path) dstdbpath := filepath.Join(dstDataDir, path)
db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...) db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)
if err != nil { if err != nil {
xerrx("open copied queue database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmQueue))) xerrx("open copied queue database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmQueue)))
return return
} }
@ -329,7 +330,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcpath := filepath.Join(srcDataDir, "queue", mp) srcpath := filepath.Join(srcDataDir, "queue", mp)
dstpath := filepath.Join(dstDataDir, "queue", mp) dstpath := filepath.Join(dstDataDir, "queue", mp)
if linked, err := linkOrCopy(srcpath, dstpath); err != nil { if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
xerrx("linking/copying queue message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xerrx("linking/copying queue message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
} else if linked { } else if linked {
nlinked++ nlinked++
} else { } else {
@ -338,9 +339,9 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil return nil
}) })
if err != nil { if err != nil {
xerrx("processing queue messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs))) xerrx("processing queue messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs)))
} else { } else {
xvlog("queue message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs))) xvlog("queue message files linked/copied", slog.Int("linked", nlinked), slog.Int("copied", ncopied), slog.Duration("duration", time.Since(tmMsgs)))
} }
// Read through all files in queue directory and warn about anything we haven't handled yet. // Read through all files in queue directory and warn about anything we haven't handled yet.
@ -348,7 +349,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcqdir := filepath.Join(srcDataDir, "queue") srcqdir := filepath.Join(srcDataDir, "queue")
err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error { err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
xerrx("walking files in queue", err, mlog.Field("srcpath", srcqpath)) xerrx("walking files in queue", err, slog.String("srcpath", srcqpath))
return nil return nil
} }
if d.IsDir() { if d.IsDir() {
@ -362,17 +363,17 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil return nil
} }
qp := filepath.Join("queue", p) qp := filepath.Join("queue", p)
xwarnx("backing up unrecognized file in queue directory", nil, mlog.Field("path", qp)) xwarnx("backing up unrecognized file in queue directory", nil, slog.String("path", qp))
backupFile(qp) backupFile(qp)
return nil return nil
}) })
if err != nil { if err != nil {
xerrx("walking queue directory (not backed up properly)", err, mlog.Field("dir", "queue"), mlog.Field("duration", time.Since(tmWalk))) xerrx("walking queue directory (not backed up properly)", err, slog.String("dir", "queue"), slog.Duration("duration", time.Since(tmWalk)))
} else { } else {
xvlog("walked queue directory", mlog.Field("duration", time.Since(tmWalk))) xvlog("walked queue directory", slog.Duration("duration", time.Since(tmWalk)))
} }
xvlog("queue backed finished", mlog.Field("duration", time.Since(tmQueue))) xvlog("queue backed finished", slog.Duration("duration", time.Since(tmQueue)))
} }
backupQueue(filepath.FromSlash("queue/index.db")) backupQueue(filepath.FromSlash("queue/index.db"))
@ -385,7 +386,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
dbpath := filepath.Join("accounts", acc.Name, "index.db") dbpath := filepath.Join("accounts", acc.Name, "index.db")
err := backupDB(acc.DB, dbpath) err := backupDB(acc.DB, dbpath)
if err != nil { if err != nil {
xerrx("copying account database", err, mlog.Field("path", dbpath), mlog.Field("duration", time.Since(tmAccount))) xerrx("copying account database", err, slog.String("path", dbpath), slog.Duration("duration", time.Since(tmAccount)))
} }
// todo: should document/check not taking a rlock on account. // todo: should document/check not taking a rlock on account.
@ -409,7 +410,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
dstdbpath := filepath.Join(dstDataDir, dbpath) dstdbpath := filepath.Join(dstDataDir, dbpath)
db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...) db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...)
if err != nil { if err != nil {
xerrx("open copied account database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmAccount))) xerrx("open copied account database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmAccount)))
return return
} }
@ -433,7 +434,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcpath := filepath.Join(srcDataDir, amp) srcpath := filepath.Join(srcDataDir, amp)
dstpath := filepath.Join(dstDataDir, amp) dstpath := filepath.Join(dstDataDir, amp)
if linked, err := linkOrCopy(srcpath, dstpath); err != nil { if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
xerrx("linking/copying account message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) xerrx("linking/copying account message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
} else if linked { } else if linked {
nlinked++ nlinked++
} else { } else {
@ -442,9 +443,9 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil return nil
}) })
if err != nil { if err != nil {
xerrx("processing account messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs))) xerrx("processing account messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs)))
} else { } else {
xvlog("account message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs))) xvlog("account message files linked/copied", slog.Int("linked", nlinked), slog.Int("copied", ncopied), slog.Duration("duration", time.Since(tmMsgs)))
} }
// Read through all files in account directory and warn about anything we haven't handled yet. // Read through all files in account directory and warn about anything we haven't handled yet.
@ -452,7 +453,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcadir := filepath.Join(srcDataDir, "accounts", acc.Name) srcadir := filepath.Join(srcDataDir, "accounts", acc.Name)
err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error { err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
xerrx("walking files in account", err, mlog.Field("srcpath", srcapath)) xerrx("walking files in account", err, slog.String("srcpath", srcapath))
return nil return nil
} }
if d.IsDir() { if d.IsDir() {
@ -472,20 +473,20 @@ func backupctl(ctx context.Context, ctl *ctl) {
} }
ap := filepath.Join("accounts", acc.Name, p) ap := filepath.Join("accounts", acc.Name, p)
if strings.HasPrefix(p, "msg"+string(filepath.Separator)) { if strings.HasPrefix(p, "msg"+string(filepath.Separator)) {
xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, mlog.Field("path", ap)) xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, slog.String("path", ap))
} else { } else {
xwarnx("backing up unrecognized file in account directory", nil, mlog.Field("path", ap)) xwarnx("backing up unrecognized file in account directory", nil, slog.String("path", ap))
} }
backupFile(ap) backupFile(ap)
return nil return nil
}) })
if err != nil { if err != nil {
xerrx("walking account directory (not backed up properly)", err, mlog.Field("srcdir", srcadir), mlog.Field("duration", time.Since(tmWalk))) xerrx("walking account directory (not backed up properly)", err, slog.String("srcdir", srcadir), slog.Duration("duration", time.Since(tmWalk)))
} else { } else {
xvlog("walked account directory", mlog.Field("duration", time.Since(tmWalk))) xvlog("walked account directory", slog.Duration("duration", time.Since(tmWalk)))
} }
xvlog("account backup finished", mlog.Field("dir", filepath.Join("accounts", acc.Name)), mlog.Field("duration", time.Since(tmAccount))) xvlog("account backup finished", slog.String("dir", filepath.Join("accounts", acc.Name)), slog.Duration("duration", time.Since(tmAccount)))
} }
// For each configured account, open it, make a copy of the database and // For each configured account, open it, make a copy of the database and
@ -493,9 +494,9 @@ func backupctl(ctx context.Context, ctl *ctl) {
// account directories when handling "all other files" below. // account directories when handling "all other files" below.
accounts := map[string]struct{}{} accounts := map[string]struct{}{}
for _, accName := range mox.Conf.Accounts() { for _, accName := range mox.Conf.Accounts() {
acc, err := store.OpenAccount(accName) acc, err := store.OpenAccount(ctl.log, accName)
if err != nil { if err != nil {
xerrx("opening account for copying (will try to copy as regular files later)", err, mlog.Field("account", accName)) xerrx("opening account for copying (will try to copy as regular files later)", err, slog.String("account", accName))
continue continue
} }
accounts[accName] = struct{}{} accounts[accName] = struct{}{}
@ -506,7 +507,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
tmWalk := time.Now() tmWalk := time.Now()
err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error { err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
xerrx("walking path", err, mlog.Field("path", srcpath)) xerrx("walking path", err, slog.String("path", srcpath))
return nil return nil
} }
@ -536,18 +537,18 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil return nil
case "lastknownversion": // Optional file, not yet handled. case "lastknownversion": // Optional file, not yet handled.
default: default:
xwarnx("backing up unrecognized file", nil, mlog.Field("path", p)) xwarnx("backing up unrecognized file", nil, slog.String("path", p))
} }
backupFile(p) backupFile(p)
return nil return nil
}) })
if err != nil { if err != nil {
xerrx("walking other files (not backed up properly)", err, mlog.Field("duration", time.Since(tmWalk))) xerrx("walking other files (not backed up properly)", err, slog.Duration("duration", time.Since(tmWalk)))
} else { } else {
xvlog("walking other files finished", mlog.Field("duration", time.Since(tmWalk))) xvlog("walking other files finished", slog.Duration("duration", time.Since(tmWalk)))
} }
xvlog("backup finished", mlog.Field("duration", time.Since(tmStart))) xvlog("backup finished", slog.Duration("duration", time.Since(tmStart)))
writer.xclose() writer.xclose()

64
ctl.go
View file

@ -16,6 +16,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
@ -35,7 +37,7 @@ type ctl struct {
conn net.Conn conn net.Conn
r *bufio.Reader // Set for first reader. r *bufio.Reader // Set for first reader.
x any // If set, errors are handled by calling panic(x) instead of log.Fatal. x any // If set, errors are handled by calling panic(x) instead of log.Fatal.
log *mlog.Log // If set, along with x, logging is done here. log mlog.Log // If set, along with x, logging is done here.
} }
// xctl opens a ctl connection. // xctl opens a ctl connection.
@ -59,7 +61,7 @@ func (c *ctl) xerror(msg string) {
if c.x == nil { if c.x == nil {
log.Fatalln(msg) log.Fatalln(msg)
} }
c.log.Debugx("ctl error", fmt.Errorf("%s", msg), mlog.Field("cmd", c.cmd)) c.log.Debugx("ctl error", fmt.Errorf("%s", msg), slog.String("cmd", c.cmd))
c.xwrite(msg) c.xwrite(msg)
panic(c.x) panic(c.x)
} }
@ -74,7 +76,7 @@ func (c *ctl) xcheck(err error, msg string) {
if c.x == nil { if c.x == nil {
log.Fatalf("%s: %s", msg, err) log.Fatalf("%s: %s", msg, err)
} }
c.log.Debugx(msg, err, mlog.Field("cmd", c.cmd)) c.log.Debugx(msg, err, slog.String("cmd", c.cmd))
fmt.Fprintf(c.conn, "%s: %s\n", msg, err) fmt.Fprintf(c.conn, "%s: %s\n", msg, err)
panic(c.x) panic(c.x)
} }
@ -160,7 +162,7 @@ type ctlwriter struct {
conn net.Conn // Ctl socket from which messages are read. conn net.Conn // Ctl socket from which messages are read.
buf []byte // Scratch buffer, for reading response. buf []byte // Scratch buffer, for reading response.
x any // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal. x any // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal.
log *mlog.Log log mlog.Log
} }
func (s *ctlwriter) Write(buf []byte) (int, error) { func (s *ctlwriter) Write(buf []byte) (int, error) {
@ -184,7 +186,7 @@ func (s *ctlwriter) xerror(msg string) {
if s.x == nil { if s.x == nil {
log.Fatalln(msg) log.Fatalln(msg)
} else { } else {
s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd)) s.log.Debugx("error", fmt.Errorf("%s", msg), slog.String("cmd", s.cmd))
panic(s.x) panic(s.x)
} }
} }
@ -196,7 +198,7 @@ func (s *ctlwriter) xcheck(err error, msg string) {
if s.x == nil { if s.x == nil {
log.Fatalf("%s: %s", msg, err) log.Fatalf("%s: %s", msg, err)
} else { } else {
s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd)) s.log.Debugx(msg, err, slog.String("cmd", s.cmd))
panic(s.x) panic(s.x)
} }
} }
@ -213,7 +215,7 @@ type ctlreader struct {
err error // If set, returned for each read. can also be io.EOF. err error // If set, returned for each read. can also be io.EOF.
npending int // Number of bytes that can still be read until a new count line must be read. npending int // Number of bytes that can still be read until a new count line must be read.
x any // If set, errors are handled with panic(x) instead of log.Fatal. x any // If set, errors are handled with panic(x) instead of log.Fatal.
log *mlog.Log // If x is set, logging goes to log. log mlog.Log // If x is set, logging goes to log.
} }
func (s *ctlreader) Read(buf []byte) (N int, Err error) { func (s *ctlreader) Read(buf []byte) (N int, Err error) {
@ -252,7 +254,7 @@ func (s *ctlreader) xerror(msg string) {
if s.x == nil { if s.x == nil {
log.Fatalln(msg) log.Fatalln(msg)
} else { } else {
s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd)) s.log.Debugx("error", fmt.Errorf("%s", msg), slog.String("cmd", s.cmd))
panic(s.x) panic(s.x)
} }
} }
@ -264,13 +266,13 @@ func (s *ctlreader) xcheck(err error, msg string) {
if s.x == nil { if s.x == nil {
log.Fatalf("%s: %s", msg, err) log.Fatalf("%s: %s", msg, err)
} else { } else {
s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd)) s.log.Debugx(msg, err, slog.String("cmd", s.cmd))
panic(s.x) panic(s.x)
} }
} }
// servectl handles requests on the unix domain socket "ctl", e.g. for graceful shutdown, local mail delivery. // servectl handles requests on the unix domain socket "ctl", e.g. for graceful shutdown, local mail delivery.
func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) { func servectl(ctx context.Context, log mlog.Log, conn net.Conn, shutdown func()) {
log.Debug("ctl connection") log.Debug("ctl connection")
var stop = struct{}{} // Sentinel value for panic and recover. var stop = struct{}{} // Sentinel value for panic and recover.
@ -280,7 +282,7 @@ func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()
if x == nil || x == stop { if x == nil || x == stop {
return return
} }
log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", ctl.cmd)) log.Error("servectl panic", slog.Any("err", x), slog.String("cmd", ctl.cmd))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Ctl) metrics.PanicInc(metrics.Ctl)
}() }()
@ -297,7 +299,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
log := ctl.log log := ctl.log
cmd := ctl.xread() cmd := ctl.xread()
ctl.cmd = cmd ctl.cmd = cmd
log.Info("ctl command", mlog.Field("cmd", cmd)) log.Info("ctl command", slog.String("cmd", cmd))
switch cmd { switch cmd {
case "stop": case "stop":
shutdown() shutdown()
@ -314,10 +316,10 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
*/ */
to := ctl.xread() to := ctl.xread()
a, addr, err := store.OpenEmail(to) a, addr, err := store.OpenEmail(ctl.log, to)
ctl.xcheck(err, "lookup destination address") ctl.xcheck(err, "lookup destination address")
msgFile, err := store.CreateMessageTemp("ctl-deliver") msgFile, err := store.CreateMessageTemp(ctl.log, "ctl-deliver")
ctl.xcheck(err, "creating temporary message file") ctl.xcheck(err, "creating temporary message file")
defer store.CloseRemoveTempFile(log, msgFile, "deliver message") defer store.CloseRemoveTempFile(log, msgFile, "deliver message")
mw := message.NewWriter(msgFile) mw := message.NewWriter(msgFile)
@ -335,7 +337,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
a.WithWLock(func() { a.WithWLock(func() {
err := a.DeliverDestination(log, addr, m, msgFile) err := a.DeliverDestination(log, addr, m, msgFile)
ctl.xcheck(err, "delivering message") ctl.xcheck(err, "delivering message")
log.Info("message delivered through ctl", mlog.Field("to", to)) log.Info("message delivered through ctl", slog.Any("to", to))
}) })
err = a.Close() err = a.Close()
@ -353,7 +355,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
account := ctl.xread() account := ctl.xread()
pw := ctl.xread() pw := ctl.xread()
acc, err := store.OpenAccount(account) acc, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "open account") ctl.xcheck(err, "open account")
defer func() { defer func() {
if acc != nil { if acc != nil {
@ -362,7 +364,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
} }
}() }()
err = acc.SetPassword(pw) err = acc.SetPassword(ctl.log, pw)
ctl.xcheck(err, "setting password") ctl.xcheck(err, "setting password")
err = acc.Close() err = acc.Close()
ctl.xcheck(err, "closing account") ctl.xcheck(err, "closing account")
@ -442,7 +444,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
ctl.xcheck(err, "parsing id") ctl.xcheck(err, "parsing id")
} }
count, err := queue.Drop(ctx, id, todomain, recipient) count, err := queue.Drop(ctx, ctl.log, id, todomain, recipient)
ctl.xcheck(err, "dropping messages from queue") ctl.xcheck(err, "dropping messages from queue")
ctl.xwrite(fmt.Sprintf("%d", count)) ctl.xwrite(fmt.Sprintf("%d", count))
ctl.xwriteok() ctl.xwriteok()
@ -586,13 +588,13 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
pkg := ctl.xread() pkg := ctl.xread()
levelstr := ctl.xread() levelstr := ctl.xread()
if levelstr == "" { if levelstr == "" {
mox.Conf.LogLevelRemove(pkg) mox.Conf.LogLevelRemove(ctl.log, pkg)
} else { } else {
level, ok := mlog.Levels[levelstr] level, ok := mlog.Levels[levelstr]
if !ok { if !ok {
ctl.xerror("bad level") ctl.xerror("bad level")
} }
mox.Conf.LogLevelSet(pkg, level) mox.Conf.LogLevelSet(ctl.log, pkg, level)
} }
ctl.xwriteok() ctl.xwriteok()
@ -603,7 +605,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
< "ok" or error < "ok" or error
*/ */
account := ctl.xread() account := ctl.xread()
acc, err := store.OpenAccount(account) acc, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "open account") ctl.xcheck(err, "open account")
defer func() { defer func() {
if acc != nil { if acc != nil {
@ -623,9 +625,9 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
dbPath := filepath.Join(basePath, acc.Name, "junkfilter.db") dbPath := filepath.Join(basePath, acc.Name, "junkfilter.db")
bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom") bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom")
err := os.Remove(dbPath) err := os.Remove(dbPath)
log.Check(err, "removing old junkfilter database file", mlog.Field("path", dbPath)) log.Check(err, "removing old junkfilter database file", slog.String("path", dbPath))
err = os.Remove(bloomPath) err = os.Remove(bloomPath)
log.Check(err, "removing old junkfilter bloom filter file", mlog.Field("path", bloomPath)) log.Check(err, "removing old junkfilter bloom filter file", slog.String("path", bloomPath))
// Open junk filter, this creates new files. // Open junk filter, this creates new files.
jf, _, err := acc.OpenJunkFilter(ctx, ctl.log) jf, _, err := acc.OpenJunkFilter(ctx, ctl.log)
@ -651,7 +653,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
return err return err
}) })
ctl.xcheck(err, "training messages") ctl.xcheck(err, "training messages")
ctl.log.Info("retrained messages", mlog.Field("total", total), mlog.Field("trained", trained)) ctl.log.Info("retrained messages", slog.Int("total", total), slog.Int("trained", trained))
// Close junk filter, marking success. // Close junk filter, marking success.
err = jf.Close() err = jf.Close()
@ -668,7 +670,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
< stream < stream
*/ */
account := ctl.xread() account := ctl.xread()
acc, err := store.OpenAccount(account) acc, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "open account") ctl.xcheck(err, "open account")
defer func() { defer func() {
if acc != nil { if acc != nil {
@ -724,7 +726,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
const batchSize = 10000 const batchSize = 10000
xfixmsgsize := func(accName string) { xfixmsgsize := func(accName string) {
acc, err := store.OpenAccount(accName) acc, err := store.OpenAccount(ctl.log, accName)
ctl.xcheck(err, "open account") ctl.xcheck(err, "open account")
defer func() { defer func() {
err := acc.Close() err := acc.Close()
@ -791,7 +793,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
m.Size = correctSize m.Size = correctSize
mr := acc.MessageReader(m) mr := acc.MessageReader(m)
part, err := message.EnsurePart(log, false, mr, m.Size) part, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil { if err != nil {
_, werr := fmt.Fprintf(w, "parsing message %d again: %v (continuing)\n", m.ID, err) _, werr := fmt.Fprintf(w, "parsing message %d again: %v (continuing)\n", m.ID, err)
ctl.xcheck(werr, "write") ctl.xcheck(werr, "write")
@ -859,7 +861,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
const batchSize = 100 const batchSize = 100
xreparseAccount := func(accName string) { xreparseAccount := func(accName string) {
acc, err := store.OpenAccount(accName) acc, err := store.OpenAccount(ctl.log, accName)
ctl.xcheck(err, "open account") ctl.xcheck(err, "open account")
defer func() { defer func() {
err := acc.Close() err := acc.Close()
@ -880,7 +882,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
return q.ForEach(func(m store.Message) error { return q.ForEach(func(m store.Message) error {
lastID = m.ID lastID = m.ID
mr := acc.MessageReader(m) mr := acc.MessageReader(m)
p, err := message.EnsurePart(log, false, mr, m.Size) p, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil { if err != nil {
_, err := fmt.Fprintf(w, "parsing message %d: %v (continuing)\n", m.ID, err) _, err := fmt.Fprintf(w, "parsing message %d: %v (continuing)\n", m.ID, err)
ctl.xcheck(err, "write") ctl.xcheck(err, "write")
@ -935,7 +937,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
w := ctl.writer() w := ctl.writer()
xreassignThreads := func(accName string) { xreassignThreads := func(accName string) {
acc, err := store.OpenAccount(accName) acc, err := store.OpenAccount(ctl.log, accName)
ctl.xcheck(err, "open account") ctl.xcheck(err, "open account")
defer func() { defer func() {
err := acc.Close() err := acc.Close()
@ -982,7 +984,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
backupctl(ctx, ctl) backupctl(ctx, ctl)
default: default:
log.Info("unrecognized command", mlog.Field("cmd", cmd)) log.Info("unrecognized command", slog.String("cmd", cmd))
ctl.xwrite("unrecognized command") ctl.xwrite("unrecognized command")
return return
} }

19
ctl_test.go
View file

@ -21,6 +21,7 @@ import (
) )
var ctxbg = context.Background() var ctxbg = context.Background()
var pkglog = mlog.New("ctl", nil)
func tcheck(t *testing.T, err error, errmsg string) { func tcheck(t *testing.T, err error, errmsg string) {
if err != nil { if err != nil {
@ -36,19 +37,17 @@ func TestCtl(t *testing.T) {
os.RemoveAll("testdata/ctl/data") os.RemoveAll("testdata/ctl/data")
mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf") mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf")
mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf") mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf")
if errs := mox.LoadConfig(ctxbg, true, false); len(errs) > 0 { if errs := mox.LoadConfig(ctxbg, pkglog, true, false); len(errs) > 0 {
t.Fatalf("loading mox config: %v", errs) t.Fatalf("loading mox config: %v", errs)
} }
defer store.Switchboard()() defer store.Switchboard()()
xlog := mlog.New("ctl")
testctl := func(fn func(clientctl *ctl)) { testctl := func(fn func(clientctl *ctl)) {
t.Helper() t.Helper()
cconn, sconn := net.Pipe() cconn, sconn := net.Pipe()
clientctl := ctl{conn: cconn, log: xlog} clientctl := ctl{conn: cconn, log: pkglog}
serverctl := ctl{conn: sconn, log: xlog} serverctl := ctl{conn: sconn, log: pkglog}
go servectlcmd(ctxbg, &serverctl, func() {}) go servectlcmd(ctxbg, &serverctl, func() {})
fn(&clientctl) fn(&clientctl)
cconn.Close() cconn.Close()
@ -148,8 +147,8 @@ func TestCtl(t *testing.T) {
}) })
// Export data, import it again // Export data, import it again
xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, nil) xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog})
xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, nil) xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog})
testctl(func(ctl *ctl) { testctl(func(ctl *ctl) {
ctlcmdImport(ctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox")) ctlcmdImport(ctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox"))
}) })
@ -167,7 +166,7 @@ func TestCtl(t *testing.T) {
ctlcmdFixmsgsize(ctl, "mjl") ctlcmdFixmsgsize(ctl, "mjl")
}) })
testctl(func(ctl *ctl) { testctl(func(ctl *ctl) {
acc, err := store.OpenAccount("mjl") acc, err := store.OpenAccount(ctl.log, "mjl")
tcheck(t, err, "open account") tcheck(t, err, "open account")
defer acc.Close() defer acc.Close()
@ -176,13 +175,13 @@ func TestCtl(t *testing.T) {
deliver := func(m *store.Message) { deliver := func(m *store.Message) {
t.Helper() t.Helper()
m.Size = int64(len(content)) m.Size = int64(len(content))
msgf, err := store.CreateMessageTemp("ctltest") msgf, err := store.CreateMessageTemp(ctl.log, "ctltest")
tcheck(t, err, "create temp file") tcheck(t, err, "create temp file")
defer os.Remove(msgf.Name()) defer os.Remove(msgf.Name())
defer msgf.Close() defer msgf.Close()
_, err = msgf.Write(content) _, err = msgf.Write(content)
tcheck(t, err, "write message file") tcheck(t, err, "write message file")
err = acc.DeliverMailbox(xlog, "Inbox", m, msgf) err = acc.DeliverMailbox(ctl.log, "Inbox", m, msgf)
tcheck(t, err, "deliver message") tcheck(t, err, "deliver message")
} }

22
dane/dane.go
View file

@ -59,6 +59,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -132,8 +134,8 @@ func (e VerifyError) Unwrap() error {
// indicate DNSSEC errors. // indicate DNSSEC errors.
// - ErrInsecure // - ErrInsecure
// - VerifyError, potentially wrapping errors from crypto/x509. // - VerifyError, potentially wrapping errors from crypto/x509.
func Dial(ctx context.Context, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) { func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) {
log := mlog.New("dane").WithContext(ctx) log := mlog.New("dane", elog)
// Split host and port. // Split host and port.
host, portstr, err := net.SplitHostPort(address) host, portstr, err := net.SplitHostPort(address)
@ -272,7 +274,7 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a
} }
var verifiedRecord adns.TLSA var verifiedRecord adns.TLSA
config := TLSClientConfig(log, records, baseDom, moreAllowedHosts, &verifiedRecord) config := TLSClientConfig(log.Logger, records, baseDom, moreAllowedHosts, &verifiedRecord)
tlsConn := tls.Client(conn, &config) tlsConn := tls.Client(conn, &config)
if err := tlsConn.HandshakeContext(ctx); err != nil { if err := tlsConn.HandshakeContext(ctx); err != nil {
conn.Close() conn.Close()
@ -295,13 +297,14 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a
// //
// If verifiedRecord is not nil, it is set to the record that was successfully // If verifiedRecord is not nil, it is set to the record that was successfully
// verified, if any. // verified, if any.
func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config { func TLSClientConfig(elog *slog.Logger, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config {
log := mlog.New("dane", elog)
return tls.Config{ return tls.Config{
ServerName: allowedHost.ASCII, // For SNI. ServerName: allowedHost.ASCII, // For SNI.
InsecureSkipVerify: true, InsecureSkipVerify: true,
VerifyConnection: func(cs tls.ConnectionState) error { VerifyConnection: func(cs tls.ConnectionState) error {
verified, record, err := Verify(log, records, cs, allowedHost, moreAllowedHosts) verified, record, err := Verify(log.Logger, records, cs, allowedHost, moreAllowedHosts)
log.Debugx("dane verification", err, mlog.Field("verified", verified), mlog.Field("record", record)) log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record))
if verified { if verified {
if verifiedRecord != nil { if verifiedRecord != nil {
*verifiedRecord = record *verifiedRecord = record
@ -332,7 +335,8 @@ func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain,
// If an error is encountered while verifying a record, e.g. for x509 // If an error is encountered while verifying a record, e.g. for x509
// trusted-anchor verification, an error may be returned, typically one or more // trusted-anchor verification, an error may be returned, typically one or more
// (wrapped) errors of type VerifyError. // (wrapped) errors of type VerifyError.
func Verify(log *mlog.Log, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) { func Verify(elog *slog.Logger, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) {
log := mlog.New("dane", elog)
metricVerify.Inc() metricVerify.Inc()
if len(records) == 0 { if len(records) == 0 {
metricVerifyErrors.Inc() metricVerifyErrors.Inc()
@ -360,7 +364,7 @@ func Verify(log *mlog.Log, records []adns.TLSA, cs tls.ConnectionState, allowedH
// errors while verifying certificates against a trust-anchor, an error can be // errors while verifying certificates against a trust-anchor, an error can be
// returned with one or more underlying x509 verification errors. A nil-nil error // returned with one or more underlying x509 verification errors. A nil-nil error
// is only returned when verified is false. // is only returned when verified is false.
func verifySingle(log *mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) { func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) {
if len(cs.PeerCertificates) == 0 { if len(cs.PeerCertificates) == 0 {
return false, fmt.Errorf("no server certificate") return false, fmt.Errorf("no server certificate")
} }
@ -513,7 +517,7 @@ func verifySingle(log *mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowed
default: default:
// Unknown, perhaps defined in the future. Not an error. // Unknown, perhaps defined in the future. Not an error.
log.Debug("unrecognized tlsa usage, skipping", mlog.Field("tlsausage", tlsa.Usage)) log.Debug("unrecognized tlsa usage, skipping", slog.Any("tlsausage", tlsa.Usage))
return false, nil return false, nil
} }
} }

8
dane/dane_test.go
View file

@ -17,9 +17,10 @@ import (
"reflect" "reflect"
"strconv" "strconv"
"sync/atomic" "sync/atomic"
"testing"
"time" "time"
"testing" "golang.org/x/exp/slog"
"github.com/mjl-/adns" "github.com/mjl-/adns"
@ -37,7 +38,8 @@ func tcheckf(t *testing.T, err error, format string, args ...any) {
// Test dialing and DANE TLS verification. // Test dialing and DANE TLS verification.
func TestDial(t *testing.T) { func TestDial(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug})
log := mlog.New("dane", nil)
// Create fake CA/trusted-anchor certificate. // Create fake CA/trusted-anchor certificate.
taTempl := x509.Certificate{ taTempl := x509.Certificate{
@ -139,7 +141,7 @@ func TestDial(t *testing.T) {
test := func(resolver dns.Resolver, expRecord adns.TLSA, expErr any) { test := func(resolver dns.Resolver, expRecord adns.TLSA, expErr any) {
t.Helper() t.Helper()
conn, record, err := Dial(context.Background(), resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages) conn, record, err := Dial(context.Background(), log.Logger, resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages)
if err == nil { if err == nil {
conn.Close() conn.Close()
} }

8
develop.txt
View file

@ -10,6 +10,14 @@ This file has notes useful for mox developers.
name/path manipulation. Do not remove/rename files that are still open. name/path manipulation. Do not remove/rename files that are still open.
- Not all code uses adns, the DNSSEC-aware resolver. Such as code that makes - Not all code uses adns, the DNSSEC-aware resolver. Such as code that makes
http requests, like mtasts and autotls/autocert. http requests, like mtasts and autotls/autocert.
- We don't have an internal/ directory, really just to prevent long paths in
the repo, and to keep all Go code matching *.go */*.go (without matching
vendor/). Part of the packages are reusable by other software. Those reusable
packages must not cause mox implementation details (such as bstore) to get out,
which would cause unexpected dependencies. Those packages also only expose the
standard slog package for logging, not our mlog package. Packages not intended
for reuse do use mlog as it is more convenient. Internally, we always use
mlog.Log to do the logging, wrapping an slog.Logger.
# TLS certificates # TLS certificates

36
dkim/dkim.go
View file

@ -24,6 +24,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -35,8 +37,6 @@ import (
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
var xlog = mlog.New("dkim")
var ( var (
metricSign = promauto.NewCounterVec( metricSign = promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
@ -123,11 +123,11 @@ type Result struct {
// todo: use some io.Writer to hash the body and the header. // todo: use some io.Writer to hash the body and the header.
// Sign returns line(s) with DKIM-Signature headers, generated according to the configuration. // Sign returns line(s) with DKIM-Signature headers, generated according to the configuration.
func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) { func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dkim", elog)
start := timeNow() start := timeNow()
defer func() { defer func() {
log.Debugx("dkim sign result", rerr, mlog.Field("localpart", localpart), mlog.Field("domain", domain), mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start))) log.Debugx("dkim sign result", rerr, slog.Any("localpart", localpart), slog.Any("domain", domain), slog.Bool("smtputf8", smtputf8), slog.Duration("duration", time.Since(start)))
}() }()
hdrs, bodyOffset, err := parseHeaders(bufio.NewReader(&moxio.AtReader{R: msg})) hdrs, bodyOffset, err := parseHeaders(bufio.NewReader(&moxio.AtReader{R: msg}))
@ -270,11 +270,11 @@ func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c co
// record should be present. // record should be present.
// //
// authentic indicates if DNS results were DNSSEC-verified. // authentic indicates if DNS results were DNSSEC-verified.
func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, authentic bool, rerr error) { func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, authentic bool, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dkim", elog)
start := timeNow() start := timeNow()
defer func() { defer func() {
log.Debugx("dkim lookup result", rerr, mlog.Field("selector", selector), mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) log.Debugx("dkim lookup result", rerr, slog.Any("selector", selector), slog.Any("domain", domain), slog.Any("status", rstatus), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start)))
}() }()
name := selector.ASCII + "._domainkey." + domain.ASCII + "." name := selector.ASCII + "._domainkey." + domain.ASCII + "."
@ -338,8 +338,8 @@ func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Dom
// verification failure is treated as actual failure. With ignoreTestMode // verification failure is treated as actual failure. With ignoreTestMode
// false, such verification failures are treated as if there is no signature by // false, such verification failures are treated as if there is no signature by
// returning StatusNone. // returning StatusNone.
func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) { func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dkim", elog)
start := timeNow() start := timeNow()
defer func() { defer func() {
duration := float64(time.Since(start)) / float64(time.Second) duration := float64(time.Since(start)) / float64(time.Second)
@ -353,10 +353,10 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
} }
if len(results) == 0 { if len(results) == 0 {
log.Debugx("dkim verify result", rerr, mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start))) log.Debugx("dkim verify result", rerr, slog.Bool("smtputf8", smtputf8), slog.Duration("duration", time.Since(start)))
} }
for _, result := range results { for _, result := range results {
log.Debugx("dkim verify result", result.Err, mlog.Field("smtputf8", smtputf8), mlog.Field("status", result.Status), mlog.Field("sig", result.Sig), mlog.Field("record", result.Record), mlog.Field("duration", time.Since(start))) log.Debugx("dkim verify result", result.Err, slog.Bool("smtputf8", smtputf8), slog.Any("status", result.Status), slog.Any("sig", result.Sig), slog.Any("record", result.Record), slog.Duration("duration", time.Since(start)))
} }
}() }()
@ -380,7 +380,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
continue continue
} }
h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig) h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, log, sig)
if err != nil { if err != nil {
results = append(results, Result{StatusPermerror, sig, nil, false, err}) results = append(results, Result{StatusPermerror, sig, nil, false, err})
continue continue
@ -394,7 +394,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
} }
br := bufio.NewReader(&moxio.AtReader{R: r, Offset: int64(bodyOffset)}) br := bufio.NewReader(&moxio.AtReader{R: r, Offset: int64(bodyOffset)})
status, txt, authentic, err := verifySignature(ctx, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode) status, txt, authentic, err := verifySignature(ctx, log.Logger, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode)
results = append(results, Result{status, sig, txt, authentic, err}) results = append(results, Result{status, sig, txt, authentic, err})
} }
return results, nil return results, nil
@ -402,7 +402,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
// check if signature is acceptable. // check if signature is acceptable.
// Only looks at the signature parameters, not at the DNS record. // Only looks at the signature parameters, not at the DNS record.
func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) { func checkSignatureParams(ctx context.Context, log mlog.Log, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) {
// "From" header is required, ../rfc/6376:2122 ../rfc/6376:2546 // "From" header is required, ../rfc/6376:2122 ../rfc/6376:2546
var from bool var from bool
for _, h := range sig.SignedHeaders { for _, h := range sig.SignedHeaders {
@ -431,7 +431,7 @@ func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, cano
if subdom.Unicode != "" { if subdom.Unicode != "" {
subdom.Unicode = "x." + subdom.Unicode subdom.Unicode = "x." + subdom.Unicode
} }
if orgDom := publicsuffix.Lookup(ctx, subdom); subdom.ASCII == orgDom.ASCII { if orgDom := publicsuffix.Lookup(ctx, log.Logger, subdom); subdom.ASCII == orgDom.ASCII {
return 0, false, false, fmt.Errorf("%w: %s", ErrTLD, sig.Domain) return 0, false, false, fmt.Errorf("%w: %s", ErrTLD, sig.Domain)
} }
@ -480,9 +480,9 @@ func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, cano
} }
// lookup the public key in the DNS and verify the signature. // lookup the public key in the DNS and verify the signature.
func verifySignature(ctx context.Context, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, bool, error) { func verifySignature(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, bool, error) {
// ../rfc/6376:2604 // ../rfc/6376:2604
status, record, _, authentic, err := Lookup(ctx, resolver, sig.Selector, sig.Domain) status, record, _, authentic, err := Lookup(ctx, elog, resolver, sig.Selector, sig.Domain)
if err != nil { if err != nil {
// todo: for temporary errors, we could pass on information so caller returns a 4.7.5 ecode, ../rfc/6376:2777 // todo: for temporary errors, we could pass on information so caller returns a 4.7.5 ecode, ../rfc/6376:2777
return status, nil, authentic, err return status, nil, authentic, err

27
dkim/dkim_test.go
View file

@ -17,8 +17,11 @@ import (
"github.com/mjl-/mox/config" "github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
) )
var pkglog = mlog.New("dkim", nil)
func policyOK(sig *Sig) error { func policyOK(sig *Sig) error {
return nil return nil
} }
@ -143,7 +146,7 @@ test
}, },
} }
results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false) results, err := Verify(context.Background(), pkglog.Logger, resolver, false, policyOK, strings.NewReader(message), false)
if err != nil { if err != nil {
t.Fatalf("dkim verify: %v", err) t.Fatalf("dkim verify: %v", err)
} }
@ -190,7 +193,7 @@ Joe.
}, },
} }
results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false) results, err := Verify(context.Background(), pkglog.Logger, resolver, false, policyOK, strings.NewReader(message), false)
if err != nil { if err != nil {
t.Fatalf("dkim verify: %v", err) t.Fatalf("dkim verify: %v", err)
} }
@ -262,7 +265,7 @@ test
} }
ctx := context.Background() ctx := context.Background()
headers, err := Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message)) headers, err := Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message))
if err != nil { if err != nil {
t.Fatalf("sign: %v", err) t.Fatalf("sign: %v", err)
} }
@ -293,7 +296,7 @@ test
nmsg := headers + message nmsg := headers + message
results, err := Verify(ctx, resolver, false, policyOK, strings.NewReader(nmsg), false) results, err := Verify(ctx, pkglog.Logger, resolver, false, policyOK, strings.NewReader(nmsg), false)
if err != nil { if err != nil {
t.Fatalf("verify: %s", err) t.Fatalf("verify: %s", err)
} }
@ -304,31 +307,31 @@ test
//log.Infof("nmsg\n%s", nmsg) //log.Infof("nmsg\n%s", nmsg)
// Multiple From headers. // Multiple From headers.
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrFrom) { if !errors.Is(err, ErrFrom) {
t.Fatalf("sign, got err %v, expected ErrFrom", err) t.Fatalf("sign, got err %v, expected ErrFrom", err)
} }
// No From header. // No From header.
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrFrom) { if !errors.Is(err, ErrFrom) {
t.Fatalf("sign, got err %v, expected ErrFrom", err) t.Fatalf("sign, got err %v, expected ErrFrom", err)
} }
// Malformed headers. // Malformed headers.
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:<mjl@mox.example>")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:<mjl@mox.example>"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
@ -408,7 +411,7 @@ test
msg = strings.ReplaceAll(msg, "\n", "\r\n") msg = strings.ReplaceAll(msg, "\n", "\r\n")
headers, err := Sign(context.Background(), "mjl", signDomain, dkimConf, false, strings.NewReader(msg)) headers, err := Sign(context.Background(), pkglog.Logger, "mjl", signDomain, dkimConf, false, strings.NewReader(msg))
if err != nil { if err != nil {
t.Fatalf("sign: %v", err) t.Fatalf("sign: %v", err)
} }
@ -425,7 +428,7 @@ test
sign() sign()
} }
results, err := Verify(context.Background(), resolver, true, policy, strings.NewReader(msg), false) results, err := Verify(context.Background(), pkglog.Logger, resolver, true, policy, strings.NewReader(msg), false)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got verify error %v, expected %v", err, expErr) t.Fatalf("got verify error %v, expected %v", err, expErr)
} }

28
dmarc/dmarc.go
View file

@ -25,9 +25,9 @@ import (
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/spf" "github.com/mjl-/mox/spf"
)
var xlog = mlog.New("dmarc") "golang.org/x/exp/slog"
)
var ( var (
metricDMARCVerify = promauto.NewHistogramVec( metricDMARCVerify = promauto.NewHistogramVec(
@ -99,11 +99,11 @@ type Result struct {
// domain is the domain with the DMARC record. // domain is the domain with the DMARC record.
// //
// rauthentic indicates if the DNS results were DNSSEC-verified. // rauthentic indicates if the DNS results were DNSSEC-verified.
func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rauthentic bool, rerr error) { func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rauthentic bool, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dmarc", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("dmarc lookup result", rerr, mlog.Field("fromdomain", from), mlog.Field("status", status), mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("duration", time.Since(start))) log.Debugx("dmarc lookup result", rerr, slog.Any("fromdomain", from), slog.Any("status", status), slog.Any("domain", domain), slog.Any("record", record), slog.Duration("duration", time.Since(start)))
}() }()
// ../rfc/7489:859 ../rfc/7489:1370 // ../rfc/7489:859 ../rfc/7489:1370
@ -114,7 +114,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status
} }
if record == nil { if record == nil {
// ../rfc/7489:761 ../rfc/7489:1377 // ../rfc/7489:761 ../rfc/7489:1377
domain = publicsuffix.Lookup(ctx, from) domain = publicsuffix.Lookup(ctx, log.Logger, from)
if domain == from { if domain == from {
return StatusNone, domain, nil, txt, authentic, err return StatusNone, domain, nil, txt, authentic, err
} }
@ -202,11 +202,11 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain
// example in RFC 7489. // example in RFC 7489.
// //
// authentic indicates if the DNS results were DNSSEC-verified. // authentic indicates if the DNS results were DNSSEC-verified.
func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) { func LookupExternalReportsAccepted(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dmarc", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start))) log.Debugx("dmarc externalreports result", rerr, slog.Bool("accepts", accepts), slog.Any("dmarcdomain", dmarcDomain), slog.Any("extdestdomain", extDestDomain), slog.Any("records", records), slog.Duration("duration", time.Since(start)))
}() }()
status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain) status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
@ -226,8 +226,8 @@ func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, d
// against the message (for inclusion in Authentication-Result headers). // against the message (for inclusion in Authentication-Result headers).
// //
// useResult indicates if the result should be applied in a policy decision. // useResult indicates if the result should be applied in a policy decision.
func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) { func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) {
log := xlog.WithContext(ctx) log := mlog.New("dmarc", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
use := "no" use := "no"
@ -239,10 +239,10 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
reject = "yes" reject = "yes"
} }
metricDMARCVerify.WithLabelValues(string(result.Status), reject, use).Observe(float64(time.Since(start)) / float64(time.Second)) metricDMARCVerify.WithLabelValues(string(result.Status), reject, use).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("dmarc verify result", result.Err, mlog.Field("fromdomain", from), mlog.Field("dkimresults", dkimResults), mlog.Field("spfresult", spfResult), mlog.Field("status", result.Status), mlog.Field("reject", result.Reject), mlog.Field("use", useResult), mlog.Field("duration", time.Since(start))) log.Debugx("dmarc verify result", result.Err, slog.Any("fromdomain", from), slog.Any("dkimresults", dkimResults), slog.Any("spfresult", spfResult), slog.Any("status", result.Status), slog.Bool("reject", result.Reject), slog.Bool("use", useResult), slog.Duration("duration", time.Since(start)))
}() }()
status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from) status, recordDomain, record, _, authentic, err := Lookup(ctx, log.Logger, resolver, from)
if record == nil { if record == nil {
return false, Result{false, status, false, false, recordDomain, record, authentic, err} return false, Result{false, status, false, false, recordDomain, record, authentic, err}
} }
@ -277,7 +277,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
if r, ok := pubsuffixes[name]; ok { if r, ok := pubsuffixes[name]; ok {
return r return r
} }
r := publicsuffix.Lookup(ctx, name) r := publicsuffix.Lookup(ctx, log.Logger, name)
pubsuffixes[name] = r pubsuffixes[name] = r
return r return r
} }

9
dmarc/dmarc_test.go
View file

@ -8,9 +8,12 @@ import (
"github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/spf" "github.com/mjl-/mox/spf"
) )
var pkglog = mlog.New("dmarc", nil)
func TestLookup(t *testing.T) { func TestLookup(t *testing.T) {
resolver := dns.MockResolver{ resolver := dns.MockResolver{
TXT: map[string][]string{ TXT: map[string][]string{
@ -29,7 +32,7 @@ func TestLookup(t *testing.T) {
test := func(d string, expStatus Status, expDomain string, expRecord *Record, expErr error) { test := func(d string, expStatus Status, expDomain string, expRecord *Record, expErr error) {
t.Helper() t.Helper()
status, dom, record, _, _, err := Lookup(context.Background(), resolver, dns.Domain{ASCII: d}) status, dom, record, _, _, err := Lookup(context.Background(), pkglog.Logger, resolver, dns.Domain{ASCII: d})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %#v, expected %#v", err, expErr) t.Fatalf("got err %#v, expected %#v", err, expErr)
} }
@ -68,7 +71,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) {
test := func(dom, extdom string, expStatus Status, expAccepts bool, expErr error) { test := func(dom, extdom string, expStatus Status, expAccepts bool, expErr error) {
t.Helper() t.Helper()
accepts, status, _, _, _, err := LookupExternalReportsAccepted(context.Background(), resolver, dns.Domain{ASCII: dom}, dns.Domain{ASCII: extdom}) accepts, status, _, _, _, err := LookupExternalReportsAccepted(context.Background(), pkglog.Logger, resolver, dns.Domain{ASCII: dom}, dns.Domain{ASCII: extdom})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %#v, expected %#v", err, expErr) t.Fatalf("got err %#v, expected %#v", err, expErr)
} }
@ -124,7 +127,7 @@ func TestVerify(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("parsing domain: %v", err) t.Fatalf("parsing domain: %v", err)
} }
useResult, result := Verify(context.Background(), resolver, from, dkimResults, spfResult, spfIdentity, true) useResult, result := Verify(context.Background(), pkglog.Logger, resolver, from, dkimResults, spfResult, spfIdentity, true)
if useResult != expUseResult || !equalResult(result, expResult) { if useResult != expUseResult || !equalResult(result, expResult) {
t.Fatalf("verify: got useResult %v, result %#v, expected %v %#v", useResult, result, expUseResult, expResult) t.Fatalf("verify: got useResult %v, result %#v, expected %v %#v", useResult, result, expUseResult, expResult)
} }

69
dmarcdb/eval.go
View file

@ -23,6 +23,7 @@ import (
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -300,13 +301,13 @@ var jitteredTimeUntil = func(t time.Time) time.Duration {
// sends DMARC reports to domains that requested them. // sends DMARC reports to domains that requested them.
func Start(resolver dns.Resolver) { func Start(resolver dns.Resolver) {
go func() { go func() {
log := mlog.New("dmarcdb") log := mlog.New("dmarcdb", nil)
defer func() { defer func() {
// In case of panic don't take the whole program down. // In case of panic don't take the whole program down.
x := recover() x := recover()
if x != nil { if x != nil {
log.Error("recover from panic", mlog.Field("panic", x)) log.Error("recover from panic", slog.Any("panic", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Dmarcdb) metrics.PanicInc(metrics.Dmarcdb)
} }
@ -358,7 +359,7 @@ func Start(resolver dns.Resolver) {
log.Check(err, "removing stale dmarc evaluations from database") log.Check(err, "removing stale dmarc evaluations from database")
clog := log.WithCid(mox.Cid()) clog := log.WithCid(mox.Cid())
clog.Info("sending dmarc aggregate reports", mlog.Field("end", nextEnd.UTC()), mlog.Field("intervals", intervals)) clog.Info("sending dmarc aggregate reports", slog.Time("end", nextEnd.UTC()), slog.Any("intervals", intervals))
if err := sendReports(ctx, clog, resolver, db, nextEnd, intervals); err != nil { if err := sendReports(ctx, clog, resolver, db, nextEnd, intervals); err != nil {
clog.Errorx("sending dmarc aggregate reports", err) clog.Errorx("sending dmarc aggregate reports", err)
metricReportError.Inc() metricReportError.Inc()
@ -393,7 +394,7 @@ var sleepBetween = func(ctx context.Context, between time.Duration) (ok bool) {
// sendReports gathers all policy domains that have evaluations that should // sendReports gathers all policy domains that have evaluations that should
// receive a DMARC report and sends a report to each. // receive a DMARC report and sends a report to each.
func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error { func sendReports(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error {
ivals := make([]any, len(intervals)) ivals := make([]any, len(intervals))
for i, v := range intervals { for i, v := range intervals {
ivals[i] = v ivals[i] = v
@ -452,14 +453,14 @@ func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *
// In case of panic don't take the whole program down. // In case of panic don't take the whole program down.
x := recover() x := recover()
if x != nil { if x != nil {
log.Error("unhandled panic in dmarcdb sendReports", mlog.Field("panic", x)) log.Error("unhandled panic in dmarcdb sendReports", slog.Any("panic", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Dmarcdb) metrics.PanicInc(metrics.Dmarcdb)
} }
}() }()
defer wg.Done() defer wg.Done()
rlog := log.WithCid(mox.Cid()).Fields(mlog.Field("domain", domain)) rlog := log.WithCid(mox.Cid()).With(slog.Any("domain", domain))
rlog.Info("sending dmarc report") rlog.Info("sending dmarc report")
if _, err := sendReportDomain(ctx, rlog, resolver, db, endTime, domain); err != nil { if _, err := sendReportDomain(ctx, rlog, resolver, db, endTime, domain); err != nil {
rlog.Errorx("sending dmarc aggregate report to domain", err) rlog.Errorx("sending dmarc aggregate report to domain", err)
@ -478,8 +479,8 @@ type recipient struct {
maxSize uint64 maxSize uint64
} }
func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) { func parseRecipient(log mlog.Log, uri dmarc.URI) (r recipient, ok bool) {
log = log.Fields(mlog.Field("uri", uri.Address)) log = log.With(slog.Any("uri", uri.Address))
u, err := url.Parse(uri.Address) u, err := url.Parse(uri.Address)
if err != nil { if err != nil {
@ -510,14 +511,14 @@ func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) {
r.maxSize *= 1024 * 1024 * 1024 * 1024 r.maxSize *= 1024 * 1024 * 1024 * 1024
case "": case "":
default: default:
log.Debug("unrecognized max size unit in dmarc record rua value", mlog.Field("unit", uri.Unit)) log.Debug("unrecognized max size unit in dmarc record rua value", slog.String("unit", uri.Unit))
return r, false return r, false
} }
return r, true return r, true
} }
func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTime time.Time, domain string) { func removeEvaluations(ctx context.Context, log mlog.Log, db *bstore.DB, endTime time.Time, domain string) {
q := bstore.QueryDB[Evaluation](ctx, db) q := bstore.QueryDB[Evaluation](ctx, db)
q.FilterLess("Evaluated", endTime) q.FilterLess("Evaluated", endTime)
q.FilterNonzero(Evaluation{PolicyDomain: domain}) q.FilterNonzero(Evaluation{PolicyDomain: domain})
@ -528,7 +529,7 @@ func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTim
// replaceable for testing. // replaceable for testing.
var queueAdd = queue.Add var queueAdd = queue.Add
func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) { func sendReportDomain(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) {
dom, err := dns.ParseDomain(domain) dom, err := dns.ParseDomain(domain)
if err != nil { if err != nil {
return false, fmt.Errorf("parsing domain for sending reports: %v", err) return false, fmt.Errorf("parsing domain for sending reports: %v", err)
@ -567,7 +568,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// evaluations regardless. We always use the latest DMARC record when sending, but // evaluations regardless. We always use the latest DMARC record when sending, but
// we'll lump all policies of the last interval into one report. // we'll lump all policies of the last interval into one report.
// ../rfc/7489:1714 // ../rfc/7489:1714
status, _, record, _, _, err := dmarc.Lookup(ctx, resolver, dom) status, _, record, _, _, err := dmarc.Lookup(ctx, log.Logger, resolver, dom)
if err != nil { if err != nil {
// todo future: we could perhaps still send this report, assuming the values we know. in case of temporary error, we could also schedule again regardless of next interval hour (we would now only retry a 24h-interval report after 24h passed). // todo future: we could perhaps still send this report, assuming the values we know. in case of temporary error, we could also schedule again regardless of next interval hour (we would now only retry a 24h-interval report after 24h passed).
// Remove records unless it was a temporary error. We'll try again next round. // Remove records unless it was a temporary error. We'll try again next round.
@ -588,8 +589,8 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// Check if domain of rua recipient has the same organizational domain as for the // Check if domain of rua recipient has the same organizational domain as for the
// evaluations. If not, we need to verify we are allowed to send. // evaluations. If not, we need to verify we are allowed to send.
ruaOrgDom := publicsuffix.Lookup(ctx, r.address.Domain) ruaOrgDom := publicsuffix.Lookup(ctx, log.Logger, r.address.Domain)
evalOrgDom := publicsuffix.Lookup(ctx, dom) evalOrgDom := publicsuffix.Lookup(ctx, log.Logger, dom)
if ruaOrgDom == evalOrgDom { if ruaOrgDom == evalOrgDom {
recipients = append(recipients, r) recipients = append(recipients, r)
@ -599,12 +600,12 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// Verify and follow addresses in other organizational domain through // Verify and follow addresses in other organizational domain through
// <policydomain>._report._dmarc.<host> lookup. // <policydomain>._report._dmarc.<host> lookup.
// ../rfc/7489:1556 // ../rfc/7489:1556
accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, resolver, evalOrgDom, r.address.Domain) accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, log.Logger, resolver, evalOrgDom, r.address.Domain)
log.Debugx("checking if rua address with different organization domain has opted into receiving dmarc reports", err, log.Debugx("checking if rua address with different organization domain has opted into receiving dmarc reports", err,
mlog.Field("policydomain", evalOrgDom), slog.Any("policydomain", evalOrgDom),
mlog.Field("destinationdomain", r.address.Domain), slog.Any("destinationdomain", r.address.Domain),
mlog.Field("accepts", accepts), slog.Bool("accepts", accepts),
mlog.Field("status", status)) slog.Any("status", status))
if status == dmarc.StatusTemperror { if status == dmarc.StatusTemperror {
// With a temporary error, we'll try to get the report the delivered anyway, // With a temporary error, we'll try to get the report the delivered anyway,
// perhaps there are multiple recipients. // perhaps there are multiple recipients.
@ -626,7 +627,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// alternative addresses and no new address specified). // alternative addresses and no new address specified).
// ../rfc/7489:1600 // ../rfc/7489:1600
foundReplacement := false foundReplacement := false
rlog := log.Fields(mlog.Field("followedaddress", uri.Address)) rlog := log.With(slog.Any("followedaddress", uri.Address))
for _, record := range records { for _, record := range records {
for _, exturi := range record.AggregateReportAddresses { for _, exturi := range record.AggregateReportAddresses {
extr, ok := parseRecipient(rlog, exturi) extr, ok := parseRecipient(rlog, exturi)
@ -634,10 +635,10 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
continue continue
} }
if extr.address.Domain != r.address.Domain { if extr.address.Domain != r.address.Domain {
rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", mlog.Field("externaladdress", extr.address)) rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", slog.Any("externaladdress", extr.address))
errors = append(errors, fmt.Sprintf("rua %s is external domain with a replacement address %s with different host", r.address, extr.address)) errors = append(errors, fmt.Sprintf("rua %s is external domain with a replacement address %s with different host", r.address, extr.address))
} else { } else {
rlog.Debug("using replacement rua address from external _report dmarc record", mlog.Field("externaladdress", extr.address)) rlog.Debug("using replacement rua address from external _report dmarc record", slog.Any("externaladdress", extr.address))
foundReplacement = true foundReplacement = true
recipients = append(recipients, extr) recipients = append(recipients, extr)
} }
@ -744,7 +745,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
report.Records = append(report.Records, rc.ReportRecord) report.Records = append(report.Records, rc.ReportRecord)
} }
reportFile, err := store.CreateMessageTemp("dmarcreportout") reportFile, err := store.CreateMessageTemp(log, "dmarcreportout")
if err != nil { if err != nil {
return false, fmt.Errorf("creating temporary file for outgoing dmarc aggregate report: %v", err) return false, fmt.Errorf("creating temporary file for outgoing dmarc aggregate report: %v", err)
} }
@ -767,7 +768,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
return true, fmt.Errorf("writing dmarc aggregate report as xml with gzip: %v", err) return true, fmt.Errorf("writing dmarc aggregate report as xml with gzip: %v", err)
} }
msgf, err := store.CreateMessageTemp("dmarcreportmsgout") msgf, err := store.CreateMessageTemp(log, "dmarcreportmsgout")
if err != nil { if err != nil {
return false, fmt.Errorf("creating temporary message file with outgoing dmarc aggregate report: %v", err) return false, fmt.Errorf("creating temporary message file with outgoing dmarc aggregate report: %v", err)
} }
@ -828,7 +829,7 @@ Period: %s - %s UTC
return false, fmt.Errorf("querying suppress list: %v", err) return false, fmt.Errorf("querying suppress list: %v", err)
} }
if exists { if exists {
log.Info("suppressing outgoing dmarc aggregate report", mlog.Field("reportingaddress", rcpt.address)) log.Info("suppressing outgoing dmarc aggregate report", slog.Any("reportingaddress", rcpt.address))
continue continue
} }
@ -853,7 +854,7 @@ Period: %s - %s UTC
log.Errorx("queueing message with dmarc aggregate report", err) log.Errorx("queueing message with dmarc aggregate report", err)
metricReportError.Inc() metricReportError.Inc()
} else { } else {
log.Debug("dmarc aggregate report queued", mlog.Field("recipient", rcpt.address)) log.Debug("dmarc aggregate report queued", slog.Any("recipient", rcpt.address))
queued = true queued = true
metricReport.Inc() metricReport.Inc()
} }
@ -873,7 +874,7 @@ Period: %s - %s UTC
return true, nil return true, nil
} }
func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { func composeAggregateReport(ctx context.Context, log mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) {
xc := message.NewComposer(mf, 100*1024*1024) xc := message.NewComposer(mf, 100*1024*1024)
defer func() { defer func() {
x := recover() x := recover()
@ -946,10 +947,10 @@ func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fro
// Though this functionality is quite underspecified, we'll do our best to send our // Though this functionality is quite underspecified, we'll do our best to send our
// an error report in case our report is too large for all recipients. // an error report in case our report is too large for all recipients.
// ../rfc/7489:1918 // ../rfc/7489:1918
func sendErrorReport(ctx context.Context, log *mlog.Log, db *bstore.DB, fromAddr smtp.Address, recipients []message.NameAddress, reportDomain dns.Domain, reportID string, reportMsgSize int64) error { func sendErrorReport(ctx context.Context, log mlog.Log, db *bstore.DB, fromAddr smtp.Address, recipients []message.NameAddress, reportDomain dns.Domain, reportID string, reportMsgSize int64) error {
log.Debug("no reporting addresses willing to accept report given size, queuing short error message") log.Debug("no reporting addresses willing to accept report given size, queuing short error message")
msgf, err := store.CreateMessageTemp("dmarcreportmsg-out") msgf, err := store.CreateMessageTemp(log, "dmarcreportmsg-out")
if err != nil { if err != nil {
return fmt.Errorf("creating temporary message file for outgoing dmarc error report: %v", err) return fmt.Errorf("creating temporary message file for outgoing dmarc error report: %v", err)
} }
@ -992,7 +993,7 @@ Submitting-URI: %s
return fmt.Errorf("querying suppress list: %v", err) return fmt.Errorf("querying suppress list: %v", err)
} }
if exists { if exists {
log.Info("suppressing outgoing dmarc error report", mlog.Field("reportingaddress", rcpt.Address)) log.Info("suppressing outgoing dmarc error report", slog.Any("reportingaddress", rcpt.Address))
continue continue
} }
@ -1006,14 +1007,14 @@ Submitting-URI: %s
log.Errorx("queueing message with dmarc error report", err) log.Errorx("queueing message with dmarc error report", err)
metricReportError.Inc() metricReportError.Inc()
} else { } else {
log.Debug("dmarc error report queued", mlog.Field("recipient", rcpt)) log.Debug("dmarc error report queued", slog.Any("recipient", rcpt))
metricReport.Inc() metricReport.Inc()
} }
} }
return nil return nil
} }
func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { func composeErrorReport(ctx context.Context, log mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) {
xc := message.NewComposer(mf, 100*1024*1024) xc := message.NewComposer(mf, 100*1024*1024)
defer func() { defer func() {
x := recover() x := recover()
@ -1058,7 +1059,7 @@ func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAdd
return msgPrefix, xc.Has8bit, xc.SMTPUTF8, messageID, nil return msgPrefix, xc.Has8bit, xc.SMTPUTF8, messageID, nil
} }
func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string { func dkimSign(ctx context.Context, log mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string {
// Add DKIM-Signature headers if we have a key for (a higher) domain than the from // Add DKIM-Signature headers if we have a key for (a higher) domain than the from
// address, which is a host name. A signature will only be useful with higher-level // address, which is a host name. A signature will only be useful with higher-level
// domains if they have a relaxed dkim check (which is the default). If the dkim // domains if they have a relaxed dkim check (which is the default). If the dkim
@ -1068,7 +1069,7 @@ func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf
for fd != zerodom { for fd != zerodom {
confDom, ok := mox.Conf.Domain(fd) confDom, ok := mox.Conf.Domain(fd)
if len(confDom.DKIM.Sign) > 0 { if len(confDom.DKIM.Sign) > 0 {
dkimHeaders, err := dkim.Sign(ctx, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf) dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf)
if err != nil { if err != nil {
log.Errorx("dkim-signing dmarc report, continuing without signature", err) log.Errorx("dkim-signing dmarc report, continuing without signature", err)
metricReportError.Inc() metricReportError.Inc()

8
dmarcdb/eval_test.go
View file

@ -13,6 +13,8 @@ import (
"testing" "testing"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dmarcrpt" "github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
@ -156,7 +158,7 @@ func TestEvaluations(t *testing.T) {
} }
func TestSendReports(t *testing.T) { func TestSendReports(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) mlog.SetConfig(map[string]slog.Level{"": slog.LevelDebug})
os.RemoveAll("../testdata/dmarcdb/data") os.RemoveAll("../testdata/dmarcdb/data")
mox.Context = ctxbg mox.Context = ctxbg
@ -294,7 +296,7 @@ func TestSendReports(t *testing.T) {
aggrAddrs := map[string]struct{}{} aggrAddrs := map[string]struct{}{}
errorAddrs := map[string]struct{}{} errorAddrs := map[string]struct{}{}
queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error { queueAdd = func(ctx context.Context, log mlog.Log, qm *queue.Msg, msgFile *os.File) error {
// Read message file. Also write copy to disk for inspection. // Read message file. Also write copy to disk for inspection.
buf, err := io.ReadAll(&moxio.AtReader{R: msgFile}) buf, err := io.ReadAll(&moxio.AtReader{R: msgFile})
tcheckf(t, err, "read report message") tcheckf(t, err, "read report message")
@ -309,7 +311,7 @@ func TestSendReports(t *testing.T) {
} else { } else {
aggrAddrs[addr] = struct{}{} aggrAddrs[addr] = struct{}{}
feedback, err = dmarcrpt.ParseMessageReport(log, msgFile) feedback, err = dmarcrpt.ParseMessageReport(log.Logger, msgFile)
tcheckf(t, err, "parsing generated report message") tcheckf(t, err, "parsing generated report message")
} }

11
dmarcrpt/parse.go
View file

@ -12,6 +12,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
@ -34,9 +36,10 @@ func ParseReport(r io.Reader) (*Feedback, error) {
// ParseMessageReport parses an aggregate feedback report from a mail message. The // ParseMessageReport parses an aggregate feedback report from a mail message. The
// maximum message size is 15MB, the maximum report size after decompression is // maximum message size is 15MB, the maximum report size after decompression is
// 20MB. // 20MB.
func ParseMessageReport(log *mlog.Log, r io.ReaderAt) (*Feedback, error) { func ParseMessageReport(elog *slog.Logger, r io.ReaderAt) (*Feedback, error) {
log := mlog.New("dmarcrpt", elog)
// ../rfc/7489:1801 // ../rfc/7489:1801
p, err := message.Parse(log, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024}) p, err := message.Parse(log.Logger, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024})
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing mail message: %s", err) return nil, fmt.Errorf("parsing mail message: %s", err)
} }
@ -44,7 +47,7 @@ func ParseMessageReport(log *mlog.Log, r io.ReaderAt) (*Feedback, error) {
return parseMessageReport(log, p) return parseMessageReport(log, p)
} }
func parseMessageReport(log *mlog.Log, p message.Part) (*Feedback, error) { func parseMessageReport(log mlog.Log, p message.Part) (*Feedback, error) {
// Pretty much any mime structure is allowed. ../rfc/7489:1861 // Pretty much any mime structure is allowed. ../rfc/7489:1861
// In practice, some parties will send the report as the only (non-multipart) // In practice, some parties will send the report as the only (non-multipart)
// content of the message. // content of the message.
@ -54,7 +57,7 @@ func parseMessageReport(log *mlog.Log, p message.Part) (*Feedback, error) {
} }
for { for {
sp, err := p.ParseNextPart(log) sp, err := p.ParseNextPart(log.Logger)
if err == io.EOF { if err == io.EOF {
return nil, ErrNoReport return nil, ErrNoReport
} }

8
dmarcrpt/parse_test.go
View file

@ -11,7 +11,7 @@ import (
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
var xlog = mlog.New("dmarcrpt") var pkglog = mlog.New("dmarcrpt", nil)
const reportExample = `<?xml version="1.0" encoding="UTF-8" ?> const reportExample = `<?xml version="1.0" encoding="UTF-8" ?>
<feedback> <feedback>
@ -137,7 +137,7 @@ func TestParseMessageReport(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("open %q: %s", p, err) t.Fatalf("open %q: %s", p, err)
} }
_, err = ParseMessageReport(xlog, f) _, err = ParseMessageReport(pkglog.Logger, f)
if err != nil { if err != nil {
t.Fatalf("ParseMessageReport: %q: %s", p, err) t.Fatalf("ParseMessageReport: %q: %s", p, err)
} }
@ -145,7 +145,7 @@ func TestParseMessageReport(t *testing.T) {
} }
// No report in a non-multipart message. // No report in a non-multipart message.
_, err = ParseMessageReport(xlog, strings.NewReader("From: <mjl@mox.example>\r\n\r\nNo report.\r\n")) _, err = ParseMessageReport(pkglog.Logger, strings.NewReader("From: <mjl@mox.example>\r\n\r\nNo report.\r\n"))
if err != ErrNoReport { if err != ErrNoReport {
t.Fatalf("message without report, got err %#v, expected ErrNoreport", err) t.Fatalf("message without report, got err %#v, expected ErrNoreport", err)
} }
@ -171,7 +171,7 @@ MIME-Version: 1.0
--===============5735553800636657282==-- --===============5735553800636657282==--
`, "\n", "\r\n") `, "\n", "\r\n")
_, err = ParseMessageReport(xlog, strings.NewReader(multipartNoreport)) _, err = ParseMessageReport(pkglog.Logger, strings.NewReader(multipartNoreport))
if err != ErrNoReport { if err != ErrNoReport {
t.Fatalf("message without report, got err %#v, expected ErrNoreport", err) t.Fatalf("message without report, got err %#v, expected ErrNoreport", err)
} }

168
dns/resolver.go
View file

@ -10,6 +10,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -22,8 +24,6 @@ import (
// todo future: change to interface that is closer to DNS. 1. expose nxdomain vs success with zero entries: nxdomain means the name does not exist for any dns resource record type, success with zero records means the name exists for other types than the requested type; 2. add ability to not follow cname records when resolving. the net resolver automatically follows cnames for LookupHost, LookupIP, LookupIPAddr. when resolving names found in mx records, we explicitly must not follow cnames. that seems impossible at the moment. 3. when looking up a cname, actually lookup the record? "net" LookupCNAME will return the requested name with no error if there is no CNAME record. because it returns the canonical name. // todo future: change to interface that is closer to DNS. 1. expose nxdomain vs success with zero entries: nxdomain means the name does not exist for any dns resource record type, success with zero records means the name exists for other types than the requested type; 2. add ability to not follow cname records when resolving. the net resolver automatically follows cnames for LookupHost, LookupIP, LookupIPAddr. when resolving names found in mx records, we explicitly must not follow cnames. that seems impossible at the moment. 3. when looking up a cname, actually lookup the record? "net" LookupCNAME will return the requested name with no error if there is no CNAME record. because it returns the canonical name.
// todo future: add option to not use anything in the cache, for the admin pages where you check the latest DNS settings, ignoring old cached info. // todo future: add option to not use anything in the cache, for the admin pages where you check the latest DNS settings, ignoring old cached info.
var xlog = mlog.New("dns")
func init() { func init() {
net.DefaultResolver.StrictErrors = true net.DefaultResolver.StrictErrors = true
} }
@ -74,6 +74,15 @@ func WithPackage(resolver Resolver, name string) Resolver {
type StrictResolver struct { type StrictResolver struct {
Pkg string // Name of subsystem that is making DNS requests, for metrics. Pkg string // Name of subsystem that is making DNS requests, for metrics.
Resolver *adns.Resolver // Where the actual lookups are done. If nil, adns.DefaultResolver is used for lookups. Resolver *adns.Resolver // Where the actual lookups are done. If nil, adns.DefaultResolver is used for lookups.
Log *slog.Logger
}
func (r StrictResolver) log() mlog.Log {
pkg := r.Pkg
if pkg == "" {
pkg = "dns"
}
return mlog.New(pkg, r.Log)
} }
var _ Resolver = StrictResolver{} var _ Resolver = StrictResolver{}
@ -133,13 +142,12 @@ func (r StrictResolver) LookupPort(ctx context.Context, network, service string)
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "port", err, start) metricLookupObserve(r.Pkg, "port", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "port"),
mlog.Field("type", "port"), slog.String("network", network),
mlog.Field("network", network), slog.String("service", service),
mlog.Field("service", service), slog.Int("resp", resp),
mlog.Field("resp", resp), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -152,13 +160,12 @@ func (r StrictResolver) LookupAddr(ctx context.Context, addr string) (resp []str
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "addr", err, start) metricLookupObserve(r.Pkg, "addr", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "addr"),
mlog.Field("type", "addr"), slog.String("addr", addr),
mlog.Field("addr", addr), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -179,13 +186,12 @@ func (r StrictResolver) LookupCNAME(ctx context.Context, host string) (resp stri
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "cname", err, start) metricLookupObserve(r.Pkg, "cname", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "cname"),
mlog.Field("type", "cname"), slog.String("host", host),
mlog.Field("host", host), slog.String("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -209,13 +215,12 @@ func (r StrictResolver) LookupHost(ctx context.Context, host string) (resp []str
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "host", err, start) metricLookupObserve(r.Pkg, "host", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "host"),
mlog.Field("type", "host"), slog.String("host", host),
mlog.Field("host", host), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -231,14 +236,13 @@ func (r StrictResolver) LookupIP(ctx context.Context, network, host string) (res
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "ip", err, start) metricLookupObserve(r.Pkg, "ip", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "ip"),
mlog.Field("type", "ip"), slog.String("network", network),
mlog.Field("network", network), slog.String("host", host),
mlog.Field("host", host), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -254,13 +258,12 @@ func (r StrictResolver) LookupIPAddr(ctx context.Context, host string) (resp []n
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "ipaddr", err, start) metricLookupObserve(r.Pkg, "ipaddr", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "ipaddr"),
mlog.Field("type", "ipaddr"), slog.String("host", host),
mlog.Field("host", host), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -276,13 +279,12 @@ func (r StrictResolver) LookupMX(ctx context.Context, name string) (resp []*net.
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "mx", err, start) metricLookupObserve(r.Pkg, "mx", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "mx"),
mlog.Field("type", "mx"), slog.String("name", name),
mlog.Field("name", name), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -298,13 +300,12 @@ func (r StrictResolver) LookupNS(ctx context.Context, name string) (resp []*net.
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "ns", err, start) metricLookupObserve(r.Pkg, "ns", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "ns"),
mlog.Field("type", "ns"), slog.String("name", name),
mlog.Field("name", name), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -320,16 +321,15 @@ func (r StrictResolver) LookupSRV(ctx context.Context, service, proto, name stri
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "srv", err, start) metricLookupObserve(r.Pkg, "srv", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "srv"),
mlog.Field("type", "srv"), slog.String("service", service),
mlog.Field("service", service), slog.String("proto", proto),
mlog.Field("proto", proto), slog.String("name", name),
mlog.Field("name", name), slog.String("resp0", resp0),
mlog.Field("resp0", resp0), slog.Any("resp1", resp1),
mlog.Field("resp1", resp1), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -345,13 +345,12 @@ func (r StrictResolver) LookupTXT(ctx context.Context, name string) (resp []stri
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "txt", err, start) metricLookupObserve(r.Pkg, "txt", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "txt"),
mlog.Field("type", "txt"), slog.String("name", name),
mlog.Field("name", name), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)
@ -367,15 +366,14 @@ func (r StrictResolver) LookupTLSA(ctx context.Context, port int, protocol, host
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookupObserve(r.Pkg, "tlsa", err, start) metricLookupObserve(r.Pkg, "tlsa", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err, r.log().WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg), slog.String("type", "tlsa"),
mlog.Field("type", "tlsa"), slog.Int("port", port),
mlog.Field("port", port), slog.String("protocol", protocol),
mlog.Field("protocol", protocol), slog.String("host", host),
mlog.Field("host", host), slog.Any("resp", resp),
mlog.Field("resp", resp), slog.Bool("authentic", result.Authentic),
mlog.Field("authentic", result.Authentic), slog.Duration("duration", time.Since(start)),
mlog.Field("duration", time.Since(start)),
) )
}() }()
defer resolveErrorHint(&err) defer resolveErrorHint(&err)

22
dnsbl/dnsbl.go
View file

@ -10,6 +10,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -17,8 +19,6 @@ import (
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
var xlog = mlog.New("dnsbl")
var ( var (
metricLookup = promauto.NewHistogramVec( metricLookup = promauto.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
@ -45,12 +45,12 @@ var (
) )
// Lookup checks if "ip" occurs in the DNS block list "zone" (e.g. dnsbl.example.org). // Lookup checks if "ip" occurs in the DNS block list "zone" (e.g. dnsbl.example.org).
func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) { func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dnsbl", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookup.WithLabelValues(zone.Name(), string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second)) metricLookup.WithLabelValues(zone.Name(), string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("dnsbl lookup result", rerr, mlog.Field("zone", zone), mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start))) log.Debugx("dnsbl lookup result", rerr, slog.Any("zone", zone), slog.Any("ip", ip), slog.Any("status", rstatus), slog.String("explanation", rexplanation), slog.Duration("duration", time.Since(start)))
}() }()
b := &strings.Builder{} b := &strings.Builder{}
@ -93,7 +93,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.
if dns.IsNotFound(err) { if dns.IsNotFound(err) {
return StatusFail, "", nil return StatusFail, "", nil
} else if err != nil { } else if err != nil {
log.Debugx("looking up txt record from dnsbl", err, mlog.Field("addr", addr)) log.Debugx("looking up txt record from dnsbl", err, slog.String("addr", addr))
return StatusFail, "", nil return StatusFail, "", nil
} }
return StatusFail, strings.Join(txts, "; "), nil return StatusFail, strings.Join(txts, "; "), nil
@ -104,16 +104,16 @@ func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.
// Users of a DNSBL should periodically check if the DNSBL is still operating // Users of a DNSBL should periodically check if the DNSBL is still operating
// properly. // properly.
// For temporary errors, ErrDNS is returned. // For temporary errors, ErrDNS is returned.
func CheckHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rerr error) { func CheckHealth(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone dns.Domain) (rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("dnsbl", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("dnsbl healthcheck result", rerr, mlog.Field("zone", zone), mlog.Field("duration", time.Since(start))) log.Debugx("dnsbl healthcheck result", rerr, slog.Any("zone", zone), slog.Duration("duration", time.Since(start)))
}() }()
// ../rfc/5782:355 // ../rfc/5782:355
status1, _, err1 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 1)) status1, _, err1 := Lookup(ctx, log.Logger, resolver, zone, net.IPv4(127, 0, 0, 1))
status2, _, err2 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 2)) status2, _, err2 := Lookup(ctx, log.Logger, resolver, zone, net.IPv4(127, 0, 0, 2))
if status1 == StatusPass && status2 == StatusFail { if status1 == StatusPass && status2 == StatusFail {
return nil return nil
} else if status1 == StatusFail { } else if status1 == StatusFail {

14
dnsbl/dnsbl_test.go
View file

@ -6,10 +6,12 @@ import (
"testing" "testing"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
) )
func TestDNSBL(t *testing.T) { func TestDNSBL(t *testing.T) {
ctx := context.Background() ctx := context.Background()
log := mlog.New("dnsbl", nil)
resolver := dns.MockResolver{ resolver := dns.MockResolver{
A: map[string][]string{ A: map[string][]string{
@ -23,7 +25,7 @@ func TestDNSBL(t *testing.T) {
}, },
} }
if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil { if status, expl, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil {
t.Fatalf("lookup: %v", err) t.Fatalf("lookup: %v", err)
} else if status != StatusFail { } else if status != StatusFail {
t.Fatalf("lookup, got status %v, expected fail", status) t.Fatalf("lookup, got status %v, expected fail", status)
@ -31,7 +33,7 @@ func TestDNSBL(t *testing.T) {
t.Fatalf("lookup, got explanation %q", expl) t.Fatalf("lookup, got explanation %q", expl)
} }
if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil { if status, expl, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil {
t.Fatalf("lookup: %v", err) t.Fatalf("lookup: %v", err)
} else if status != StatusFail { } else if status != StatusFail {
t.Fatalf("lookup, got status %v, expected fail", status) t.Fatalf("lookup, got status %v, expected fail", status)
@ -39,17 +41,17 @@ func TestDNSBL(t *testing.T) {
t.Fatalf("lookup, got explanation %q", expl) t.Fatalf("lookup, got explanation %q", expl)
} }
if status, _, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil { if status, _, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil {
t.Fatalf("lookup: %v", err) t.Fatalf("lookup: %v", err)
} else if status != StatusPass { } else if status != StatusPass {
t.Fatalf("lookup, got status %v, expected pass", status) t.Fatalf("lookup, got status %v, expected pass", status)
} }
// ../rfc/5782:357 // ../rfc/5782:357
if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.com"}); err != nil { if err := CheckHealth(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}); err != nil {
t.Fatalf("dnsbl not healthy: %v", err) t.Fatalf("dnsbl not healthy: %v", err)
} }
if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.org"}); err == nil { if err := CheckHealth(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.org"}); err == nil {
t.Fatalf("bad dnsbl is healthy") t.Fatalf("bad dnsbl is healthy")
} }
@ -58,7 +60,7 @@ func TestDNSBL(t *testing.T) {
"1.0.0.127.example.com.": {"127.0.0.2"}, // Should not be present in healthy dnsbl. "1.0.0.127.example.com.": {"127.0.0.2"}, // Should not be present in healthy dnsbl.
}, },
} }
if err := CheckHealth(ctx, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil { if err := CheckHealth(ctx, log.Logger, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil {
t.Fatalf("bad dnsbl is healthy") t.Fatalf("bad dnsbl is healthy")
} }
} }

8
dsn/dsn.go
View file

@ -16,6 +16,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
@ -135,7 +137,7 @@ type Recipient struct {
// DSN. // DSN.
// //
// DKIM signatures are added if DKIM signing is configured for the "from" domain. // DKIM signatures are added if DKIM signing is configured for the "from" domain.
func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) { func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) {
// ../rfc/3462:119 // ../rfc/3462:119
// ../rfc/3464:377 // ../rfc/3464:377
// We'll make a multipart/report with 2 or 3 parts: // We'll make a multipart/report with 2 or 3 parts:
@ -381,9 +383,9 @@ func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) {
continue continue
} }
dkimHeaders, err := dkim.Sign(context.Background(), m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data)) dkimHeaders, err := dkim.Sign(context.Background(), log.Logger, m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data))
if err != nil { if err != nil {
log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, mlog.Field("domain", fd)) log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, slog.Any("domain", fd))
} else { } else {
data = append([]byte(dkimHeaders), data...) data = append([]byte(dkimHeaders), data...)
} }

8
dsn/dsn_test.go
View file

@ -20,7 +20,7 @@ import (
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
var xlog = mlog.New("dsn") var pkglog = mlog.New("dsn", nil)
func xparseDomain(s string) dns.Domain { func xparseDomain(s string) dns.Domain {
d, err := dns.ParseDomain(s) d, err := dns.ParseDomain(s)
@ -36,7 +36,7 @@ func xparseIPDomain(s string) dns.IPDomain {
func tparseMessage(t *testing.T, data []byte, nparts int) (*Message, *message.Part) { func tparseMessage(t *testing.T, data []byte, nparts int) (*Message, *message.Part) {
t.Helper() t.Helper()
m, p, err := Parse(xlog, bytes.NewReader(data)) m, p, err := Parse(pkglog.Logger, bytes.NewReader(data))
if err != nil { if err != nil {
t.Fatalf("parsing dsn: %v", err) t.Fatalf("parsing dsn: %v", err)
} }
@ -75,7 +75,7 @@ func tcompareReader(t *testing.T, r io.Reader, exp []byte) {
} }
func TestDSN(t *testing.T) { func TestDSN(t *testing.T) {
log := mlog.New("dsn") log := mlog.New("dsn", nil)
now := time.Now() now := time.Now()
@ -143,7 +143,7 @@ func TestDSN(t *testing.T) {
"testsel._domainkey.mox.example.": {"v=DKIM1;h=sha256;t=s;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3ZId3ys70VFspp/VMFaxMOrNjHNPg04NOE1iShih16b3Ex7hHBOgC1UvTGSmrMlbCB1OxTXkvf6jW6S4oYRnZYVNygH6zKUwYYhaSaGIg1xA/fDn+IgcTRyLoXizMUgUgpTGyxhNrwIIWv+i7jjbs3TKpP3NU4owQ/rxowmSNqg+fHIF1likSvXvljYS" + "jaFXXnWfYibW7TdDCFFpN4sB5o13+as0u4vLw6MvOi59B1tLype1LcHpi1b9PfxNtznTTdet3kL0paxIcWtKHT0LDPUos8YYmiPa5nGbUqlC7d+4YT2jQPvwGxCws1oo2Tw6nj1UaihneYGAyvEky49FBwIDAQAB"}, "testsel._domainkey.mox.example.": {"v=DKIM1;h=sha256;t=s;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3ZId3ys70VFspp/VMFaxMOrNjHNPg04NOE1iShih16b3Ex7hHBOgC1UvTGSmrMlbCB1OxTXkvf6jW6S4oYRnZYVNygH6zKUwYYhaSaGIg1xA/fDn+IgcTRyLoXizMUgUgpTGyxhNrwIIWv+i7jjbs3TKpP3NU4owQ/rxowmSNqg+fHIF1likSvXvljYS" + "jaFXXnWfYibW7TdDCFFpN4sB5o13+as0u4vLw6MvOi59B1tLype1LcHpi1b9PfxNtznTTdet3kL0paxIcWtKHT0LDPUos8YYmiPa5nGbUqlC7d+4YT2jQPvwGxCws1oo2Tw6nj1UaihneYGAyvEky49FBwIDAQAB"},
}, },
} }
results, err := dkim.Verify(context.Background(), resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false) results, err := dkim.Verify(context.Background(), log.Logger, resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false)
if err != nil { if err != nil {
t.Fatalf("dkim verify: %v", err) t.Fatalf("dkim verify: %v", err)
} }

10
dsn/parse.go
View file

@ -9,6 +9,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
@ -23,17 +25,19 @@ import (
// The first return value is the machine-parsed DSN message. The second value is // The first return value is the machine-parsed DSN message. The second value is
// the entire MIME multipart message. Use its Parts field to access the // the entire MIME multipart message. Use its Parts field to access the
// human-readable text and optional original message/headers. // human-readable text and optional original message/headers.
func Parse(log *mlog.Log, r io.ReaderAt) (*Message, *message.Part, error) { func Parse(elog *slog.Logger, r io.ReaderAt) (*Message, *message.Part, error) {
log := mlog.New("dsn", elog)
// DSNs can mix and match subtypes with and without utf-8. ../rfc/6533:441 // DSNs can mix and match subtypes with and without utf-8. ../rfc/6533:441
part, err := message.Parse(log, false, r) part, err := message.Parse(log.Logger, false, r)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("parsing message: %v", err) return nil, nil, fmt.Errorf("parsing message: %v", err)
} }
if part.MediaType != "MULTIPART" || part.MediaSubType != "REPORT" { if part.MediaType != "MULTIPART" || part.MediaSubType != "REPORT" {
return nil, nil, fmt.Errorf(`message has content-type %q, must have "message/report"`, strings.ToLower(part.MediaType+"/"+part.MediaSubType)) return nil, nil, fmt.Errorf(`message has content-type %q, must have "message/report"`, strings.ToLower(part.MediaType+"/"+part.MediaSubType))
} }
err = part.Walk(log, nil) err = part.Walk(log.Logger, nil)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("parsing message parts: %v", err) return nil, nil, fmt.Errorf("parsing message parts: %v", err)
} }

3
export.go
View file

@ -8,7 +8,6 @@ import (
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -66,7 +65,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
}() }()
a := store.DirArchiver{Dir: dst} a := store.DirArchiver{Dir: dst}
err = store.ExportMessages(context.Background(), mlog.New("export"), db, accountDir, a, !mbox, mailbox) err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox)
xcheckf(err, "exporting messages") xcheckf(err, "exporting messages")
err = a.Close() err = a.Close()
xcheckf(err, "closing archiver") xcheckf(err, "closing archiver")

23
gentestdata.go
View file

@ -54,7 +54,6 @@ func cmdGentestdata(c *cmd) {
return f return f
} }
log := mlog.New("gentestdata")
ctxbg := context.Background() ctxbg := context.Background()
mox.Conf.Log[""] = mlog.LevelInfo mox.Conf.Log[""] = mlog.LevelInfo
mlog.SetConfig(mox.Conf.Log) mlog.SetConfig(mox.Conf.Log)
@ -217,7 +216,7 @@ Accounts:
xcheckf(err, "tlsrptdb init") xcheckf(err, "tlsrptdb init")
tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport)) tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
xcheckf(err, "parsing tls report") xcheckf(err, "parsing tls report")
err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr) err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr)
xcheckf(err, "adding tls report") xcheckf(err, "adding tls report")
// Populate queue, with a message. // Populate queue, with a message.
@ -234,22 +233,22 @@ Accounts:
_, err = fmt.Fprint(mf, qmsg) _, err = fmt.Fprint(mf, qmsg)
xcheckf(err, "writing message") xcheckf(err, "writing message")
qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil) qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
err = queue.Add(ctxbg, log, &qm, mf) err = queue.Add(ctxbg, c.log, &qm, mf)
xcheckf(err, "enqueue message") xcheckf(err, "enqueue message")
// Create three accounts. // Create three accounts.
// First account without messages. // First account without messages.
accTest0, err := store.OpenAccount("test0") accTest0, err := store.OpenAccount(c.log, "test0")
xcheckf(err, "open account test0") xcheckf(err, "open account test0")
err = accTest0.ThreadingWait(log) err = accTest0.ThreadingWait(c.log)
xcheckf(err, "wait for threading to finish") xcheckf(err, "wait for threading to finish")
err = accTest0.Close() err = accTest0.Close()
xcheckf(err, "close account") xcheckf(err, "close account")
// Second account with one message. // Second account with one message.
accTest1, err := store.OpenAccount("test1") accTest1, err := store.OpenAccount(c.log, "test1")
xcheckf(err, "open account test1") xcheckf(err, "open account test1")
err = accTest1.ThreadingWait(log) err = accTest1.ThreadingWait(c.log)
xcheckf(err, "wait for threading to finish") xcheckf(err, "wait for threading to finish")
err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error { err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
@ -285,7 +284,7 @@ Accounts:
xcheckf(err, "creating temp file for delivery") xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf, msg) _, err = fmt.Fprint(mf, msg)
xcheckf(err, "writing deliver message to file") xcheckf(err, "writing deliver message to file")
err = accTest1.DeliverMessage(log, tx, &m, mf, false, true, false) err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false)
mfname := mf.Name() mfname := mf.Name()
xcheckf(err, "add message to account test1") xcheckf(err, "add message to account test1")
@ -307,9 +306,9 @@ Accounts:
xcheckf(err, "close account") xcheckf(err, "close account")
// Third account with two messages and junkfilter. // Third account with two messages and junkfilter.
accTest2, err := store.OpenAccount("test2") accTest2, err := store.OpenAccount(c.log, "test2")
xcheckf(err, "open account test2") xcheckf(err, "open account test2")
err = accTest2.ThreadingWait(log) err = accTest2.ThreadingWait(c.log)
xcheckf(err, "wait for threading to finish") xcheckf(err, "wait for threading to finish")
err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error { err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
@ -345,7 +344,7 @@ Accounts:
xcheckf(err, "creating temp file for delivery") xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf0, msg0) _, err = fmt.Fprint(mf0, msg0)
xcheckf(err, "writing deliver message to file") xcheckf(err, "writing deliver message to file")
err = accTest2.DeliverMessage(log, tx, &m0, mf0, false, false, false) err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false)
xcheckf(err, "add message to account test2") xcheckf(err, "add message to account test2")
mf0name := mf0.Name() mf0name := mf0.Name()
@ -376,7 +375,7 @@ Accounts:
xcheckf(err, "creating temp file for delivery") xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf1, msg1) _, err = fmt.Fprint(mf1, msg1)
xcheckf(err, "writing deliver message to file") xcheckf(err, "writing deliver message to file")
err = accTest2.DeliverMessage(log, tx, &m1, mf1, false, false, false) err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false)
xcheckf(err, "add message to account test2") xcheckf(err, "add message to account test2")
mf1name := mf1.Name() mf1name := mf1.Name()

11
http/autoconf.go
View file

@ -6,11 +6,12 @@ import (
"net/http" "net/http"
"strings" "strings"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
"rsc.io/qr" "rsc.io/qr"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
@ -55,7 +56,7 @@ var (
// User should create a DNS record: autoconfig.<domain> (CNAME or A). // User should create a DNS record: autoconfig.<domain> (CNAME or A).
// See https://wiki.mozilla.org/Thunderbird:Autoconfiguration:ConfigFileFormat // See https://wiki.mozilla.org/Thunderbird:Autoconfiguration:ConfigFileFormat
func autoconfHandle(w http.ResponseWriter, r *http.Request) { func autoconfHandle(w http.ResponseWriter, r *http.Request) {
log := xlog.WithContext(r.Context()) log := pkglog.WithContext(r.Context())
var addrDom string var addrDom string
defer func() { defer func() {
@ -63,7 +64,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
}() }()
email := r.FormValue("emailaddress") email := r.FormValue("emailaddress")
log.Debug("autoconfig request", mlog.Field("email", email)) log.Debug("autoconfig request", slog.String("email", email))
addr, err := smtp.ParseAddress(email) addr, err := smtp.ParseAddress(email)
if err != nil { if err != nil {
http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest) http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest)
@ -143,7 +144,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
// //
// Thunderbird does understand autodiscover. // Thunderbird does understand autodiscover.
func autodiscoverHandle(w http.ResponseWriter, r *http.Request) { func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
log := xlog.WithContext(r.Context()) log := pkglog.WithContext(r.Context())
var addrDom string var addrDom string
defer func() { defer func() {
@ -161,7 +162,7 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
return return
} }
log.Debug("autodiscover request", mlog.Field("email", req.Request.EmailAddress)) log.Debug("autodiscover request", slog.String("email", req.Request.EmailAddress))
addr, err := smtp.ParseAddress(req.Request.EmailAddress) addr, err := smtp.ParseAddress(req.Request.EmailAddress)
if err != nil { if err != nil {

22
http/gzcache.go
View file

@ -16,6 +16,8 @@ import (
"sync" "sync"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
@ -75,7 +77,7 @@ func loadStaticGzipCache(dir string, maxSize int64) {
os.MkdirAll(dir, 0700) os.MkdirAll(dir, 0700)
entries, err := os.ReadDir(dir) entries, err := os.ReadDir(dir)
if err != nil && !os.IsNotExist(err) { if err != nil && !os.IsNotExist(err) {
xlog.Errorx("listing static gzip cache files", err, mlog.Field("dir", dir)) pkglog.Errorx("listing static gzip cache files", err, slog.String("dir", dir))
} }
for _, e := range entries { for _, e := range entries {
name := e.Name() name := e.Name()
@ -111,9 +113,9 @@ func loadStaticGzipCache(dir string, maxSize int64) {
atime, err = statAtime(fi.Sys()) atime, err = statAtime(fi.Sys())
} }
if err != nil { if err != nil {
xlog.Infox("removing unusable/unrecognized file in static gzip cache dir", err) pkglog.Infox("removing unusable/unrecognized file in static gzip cache dir", err)
xerr := os.Remove(filepath.Join(dir, name)) xerr := os.Remove(filepath.Join(dir, name))
xlog.Check(xerr, "removing unusable file in static gzip cache dir", mlog.Field("error", err), mlog.Field("dir", dir), mlog.Field("filename", name)) pkglog.Check(xerr, "removing unusable file in static gzip cache dir", slog.Any("error", err), slog.String("dir", dir), slog.String("filename", name))
continue continue
} }
staticgzcache.paths[path] = gzfile{ staticgzcache.paths[path] = gzfile{
@ -163,7 +165,7 @@ func (c *gzcache) evictPath(path string) {
c.unlink(gf.use) c.unlink(gf.use)
c.size -= gf.gzsize c.size -= gf.gzsize
err := os.Remove(staticCachePath(c.dir, path, gf.mtime)) err := os.Remove(staticCachePath(c.dir, path, gf.mtime))
xlog.Check(err, "removing cached gzipped static file", mlog.Field("path", path)) pkglog.Check(err, "removing cached gzipped static file", slog.String("path", path))
} }
// Open cached file for path, requiring it has mtime. If there is no usable cached // Open cached file for path, requiring it has mtime. If there is no usable cached
@ -189,7 +191,7 @@ func (c *gzcache) openPath(path string, mtime int64) (*os.File, int64) {
p := staticCachePath(c.dir, path, gf.mtime) p := staticCachePath(c.dir, path, gf.mtime)
f, err := os.Open(p) f, err := os.Open(p)
if err != nil { if err != nil {
xlog.Errorx("open static cached gzip file, removing from cache", err, mlog.Field("path", path)) pkglog.Errorx("open static cached gzip file, removing from cache", err, slog.String("path", path))
// Perhaps someone removed the file? Remove from cache, it will be recreated. // Perhaps someone removed the file? Remove from cache, it will be recreated.
c.evictPath(path) c.evictPath(path)
return nil, 0 return nil, 0
@ -303,8 +305,8 @@ type staticgzcacheReplacer struct {
handled bool handled bool
} }
func (w *staticgzcacheReplacer) logger() *mlog.Log { func (w *staticgzcacheReplacer) logger() mlog.Log {
return xlog.WithContext(w.r.Context()) return pkglog.WithContext(w.r.Context())
} }
// Header returns the header of the underlying ResponseWriter. // Header returns the header of the underlying ResponseWriter.
@ -353,7 +355,7 @@ func (w *staticgzcacheReplacer) WriteHeader(statusCode int) {
p := staticCachePath(staticgzcache.dir, w.uncomprPath, w.uncomprMtime.UnixNano()) p := staticCachePath(staticgzcache.dir, w.uncomprPath, w.uncomprMtime.UnixNano())
ngzf, err := os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) ngzf, err := os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
if err != nil { if err != nil {
w.logger().Errorx("create new static gzip cache file", err, mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p)) w.logger().Errorx("create new static gzip cache file", err, slog.String("requestpath", w.uncomprPath), slog.String("fspath", p))
staticgzcache.abortPath(w.uncomprPath) staticgzcache.abortPath(w.uncomprPath)
return return
} }
@ -361,9 +363,9 @@ func (w *staticgzcacheReplacer) WriteHeader(statusCode int) {
if ngzf != nil { if ngzf != nil {
staticgzcache.abortPath(w.uncomprPath) staticgzcache.abortPath(w.uncomprPath)
err := ngzf.Close() err := ngzf.Close()
w.logger().Check(err, "closing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p)) w.logger().Check(err, "closing failed static gzip cache file", slog.String("requestpath", w.uncomprPath), slog.String("fspath", p))
err = os.Remove(p) err = os.Remove(p)
w.logger().Check(err, "removing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p)) w.logger().Check(err, "removing failed static gzip cache file", slog.String("requestpath", w.uncomprPath), slog.String("fspath", p))
} }
}() }()

10
http/mtasts.go
View file

@ -6,6 +6,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
@ -13,8 +15,8 @@ import (
) )
func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) { func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
log := func() *mlog.Log { log := func() mlog.Log {
return xlog.WithContext(r.Context()) return pkglog.WithContext(r.Context())
} }
host := strings.ToLower(r.Host) host := strings.ToLower(r.Host)
@ -30,7 +32,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
} }
domain, err := dns.ParseDomain(host) domain, err := dns.ParseDomain(host)
if err != nil { if err != nil {
log().Errorx("mtasts policy request: bad domain", err, mlog.Field("host", host)) log().Errorx("mtasts policy request: bad domain", err, slog.String("host", host))
http.NotFound(w, r) http.NotFound(w, r)
return return
} }
@ -51,7 +53,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
} }
d, err := dns.ParseDomain(s) d, err := dns.ParseDomain(s)
if err != nil { if err != nil {
log().Errorx("bad domain in mtasts config", err, mlog.Field("domain", s)) log().Errorx("bad domain in mtasts config", err, slog.String("domain", s))
http.Error(w, "500 - internal server error - invalid domain in configuration", http.StatusInternalServerError) http.Error(w, "500 - internal server error - invalid domain in configuration", http.StatusInternalServerError)
return return
} }

87
http/web.go
View file

@ -21,6 +21,7 @@ import (
_ "net/http/pprof" _ "net/http/pprof"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -37,7 +38,7 @@ import (
"github.com/mjl-/mox/webmail" "github.com/mjl-/mox/webmail"
) )
var xlog = mlog.New("http") var pkglog = mlog.New("http", nil)
var ( var (
// metricRequest tracks performance (time to write response header) of server. // metricRequest tracks performance (time to write response header) of server.
@ -96,11 +97,11 @@ type loggingWriter struct {
Err error Err error
WebsocketResponse bool // If this was a successful websocket connection with backend. WebsocketResponse bool // If this was a successful websocket connection with backend.
SizeFromClient, SizeToClient int64 // Websocket data. SizeFromClient, SizeToClient int64 // Websocket data.
Fields []mlog.Pair // Additional fields to log. Attrs []slog.Attr // Additional fields to log.
} }
func (w *loggingWriter) AddField(p mlog.Pair) { func (w *loggingWriter) AddAttr(a slog.Attr) {
w.Fields = append(w.Fields, p) w.Attrs = append(w.Attrs, a)
} }
func (w *loggingWriter) Flush() { func (w *loggingWriter) Flush() {
@ -310,43 +311,43 @@ func (w *loggingWriter) Done() {
if err == nil { if err == nil {
err = w.R.Context().Err() err = w.R.Context().Err()
} }
fields := []mlog.Pair{ attrs := []slog.Attr{
mlog.Field("httpaccess", ""), slog.String("httpaccess", ""),
mlog.Field("handler", w.Handler), slog.String("handler", w.Handler),
mlog.Field("method", method), slog.String("method", method),
mlog.Field("url", w.R.URL), slog.Any("url", w.R.URL),
mlog.Field("host", w.R.Host), slog.String("host", w.R.Host),
mlog.Field("duration", time.Since(w.Start)), slog.Duration("duration", time.Since(w.Start)),
mlog.Field("statuscode", w.StatusCode), slog.Int("statuscode", w.StatusCode),
mlog.Field("proto", strings.ToLower(w.R.Proto)), slog.String("proto", strings.ToLower(w.R.Proto)),
mlog.Field("remoteaddr", w.R.RemoteAddr), slog.Any("remoteaddr", w.R.RemoteAddr),
mlog.Field("tlsinfo", tlsinfo), slog.String("tlsinfo", tlsinfo),
mlog.Field("useragent", w.R.Header.Get("User-Agent")), slog.String("useragent", w.R.Header.Get("User-Agent")),
mlog.Field("referrr", w.R.Header.Get("Referrer")), slog.String("referrr", w.R.Header.Get("Referrer")),
} }
if w.WebsocketRequest { if w.WebsocketRequest {
fields = append(fields, attrs = append(attrs,
mlog.Field("websocketrequest", true), slog.Bool("websocketrequest", true),
) )
} }
if w.WebsocketResponse { if w.WebsocketResponse {
fields = append(fields, attrs = append(attrs,
mlog.Field("websocket", true), slog.Bool("websocket", true),
mlog.Field("sizetoclient", w.SizeToClient), slog.Int64("sizetoclient", w.SizeToClient),
mlog.Field("sizefromclient", w.SizeFromClient), slog.Int64("sizefromclient", w.SizeFromClient),
) )
} else if w.UncompressedSize > 0 { } else if w.UncompressedSize > 0 {
fields = append(fields, attrs = append(attrs,
mlog.Field("size", w.Size), slog.Int64("size", w.Size),
mlog.Field("uncompressedsize", w.UncompressedSize), slog.Int64("uncompressedsize", w.UncompressedSize),
) )
} else { } else {
fields = append(fields, attrs = append(attrs,
mlog.Field("size", w.Size), slog.Int64("size", w.Size),
) )
} }
fields = append(fields, w.Fields...) attrs = append(attrs, w.Attrs...)
xlog.WithContext(w.R.Context()).Debugx("http request", err, fields...) pkglog.WithContext(w.R.Context()).Debugx("http request", err, attrs...)
} }
// Set some http headers that should prevent potential abuse. Better safe than sorry. // Set some http headers that should prevent potential abuse. Better safe than sorry.
@ -405,9 +406,9 @@ func (s *serve) ServeHTTP(xw http.ResponseWriter, r *http.Request) {
// Rate limiting as early as possible. // Rate limiting as early as possible.
ipstr, _, err := net.SplitHostPort(r.RemoteAddr) ipstr, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil { if err != nil {
xlog.Debugx("split host:port client remoteaddr", err, mlog.Field("remoteaddr", r.RemoteAddr)) pkglog.Debugx("split host:port client remoteaddr", err, slog.Any("remoteaddr", r.RemoteAddr))
} else if ip := net.ParseIP(ipstr); ip == nil { } else if ip := net.ParseIP(ipstr); ip == nil {
xlog.Debug("parsing ip for client remoteaddr", mlog.Field("remoteaddr", r.RemoteAddr)) pkglog.Debug("parsing ip for client remoteaddr", slog.Any("remoteaddr", r.RemoteAddr))
} else if !limiterConnectionrate.Add(ip, now, 1) { } else if !limiterConnectionrate.Add(ip, now, 1) {
method := metricHTTPMethod(r.Method) method := metricHTTPMethod(r.Method)
proto := "http" proto := "http"
@ -649,7 +650,7 @@ func Listen() {
// Importing net/http/pprof registers handlers on the default serve mux. // Importing net/http/pprof registers handlers on the default serve mux.
port := config.Port(l.PprofHTTP.Port, 8011) port := config.Port(l.PprofHTTP.Port, 8011)
if _, ok := portServe[port]; ok { if _, ok := portServe[port]; ok {
xlog.Fatal("cannot serve pprof on same endpoint as other http services") pkglog.Fatal("cannot serve pprof on same endpoint as other http services")
} }
srv := &serve{[]string{"pprof-http"}, nil, nil, false} srv := &serve{[]string{"pprof-http"}, nil, nil, false}
portServe[port] = srv portServe[port] = srv
@ -686,7 +687,7 @@ func Listen() {
// presence of TLS certificates for. // presence of TLS certificates for.
for _, name := range mox.Conf.Domains() { for _, name := range mox.Conf.Domains() {
if dom, err := dns.ParseDomain(name); err != nil { if dom, err := dns.ParseDomain(name); err != nil {
xlog.Errorx("parsing domain from config", err) pkglog.Errorx("parsing domain from config", err)
} else if d, _ := mox.Conf.Domain(dom); d.DMARC != nil && d.DMARC.Domain != "" && d.DMARC.DNSDomain != dom { } else if d, _ := mox.Conf.Domain(dom); d.DMARC != nil && d.DMARC.Domain != "" && d.DMARC.DNSDomain != dom {
// Do not gather autoconfig name if this domain is configured to process reports // Do not gather autoconfig name if this domain is configured to process reports
// for domains hosted elsewhere. // for domains hosted elsewhere.
@ -695,7 +696,7 @@ func Listen() {
autoconfdom, err := dns.ParseDomain("autoconfig." + name) autoconfdom, err := dns.ParseDomain("autoconfig." + name)
if err != nil { if err != nil {
xlog.Errorx("parsing domain from config for autoconfig", err) pkglog.Errorx("parsing domain from config for autoconfig", err)
} else { } else {
hosts[autoconfdom] = struct{}{} hosts[autoconfdom] = struct{}{}
} }
@ -745,20 +746,20 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st
if tlsConfig == nil { if tlsConfig == nil {
protocol = "http" protocol = "http"
if os.Getuid() == 0 { if os.Getuid() == 0 {
xlog.Print("http listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr)) pkglog.Print("http listener", slog.String("name", name), slog.String("kinds", strings.Join(kinds, ",")), slog.String("address", addr))
} }
ln, err = mox.Listen(mox.Network(ip), addr) ln, err = mox.Listen(mox.Network(ip), addr)
if err != nil { if err != nil {
xlog.Fatalx("http: listen", err, mlog.Field("addr", addr)) pkglog.Fatalx("http: listen", err, slog.Any("addr", addr))
} }
} else { } else {
protocol = "https" protocol = "https"
if os.Getuid() == 0 { if os.Getuid() == 0 {
xlog.Print("https listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr)) pkglog.Print("https listener", slog.String("name", name), slog.String("kinds", strings.Join(kinds, ",")), slog.String("address", addr))
} }
ln, err = mox.Listen(mox.Network(ip), addr) ln, err = mox.Listen(mox.Network(ip), addr)
if err != nil { if err != nil {
xlog.Fatalx("https: listen", err, mlog.Field("addr", addr)) pkglog.Fatalx("https: listen", err, slog.String("addr", addr))
} }
ln = tls.NewListener(ln, tlsConfig) ln = tls.NewListener(ln, tlsConfig)
} }
@ -768,11 +769,11 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st
TLSConfig: tlsConfig, TLSConfig: tlsConfig,
ReadHeaderTimeout: 30 * time.Second, ReadHeaderTimeout: 30 * time.Second,
IdleTimeout: 65 * time.Second, // Chrome closes connections after 60 seconds, firefox after 115 seconds. IdleTimeout: 65 * time.Second, // Chrome closes connections after 60 seconds, firefox after 115 seconds.
ErrorLog: golog.New(mlog.ErrWriter(xlog.Fields(mlog.Field("pkg", "net/http")), mlog.LevelInfo, protocol+" error"), "", 0), ErrorLog: golog.New(mlog.LogWriter(pkglog.With(slog.String("pkg", "net/http")), slog.LevelInfo, protocol+" error"), "", 0),
} }
serve := func() { serve := func() {
err := server.Serve(ln) err := server.Serve(ln)
xlog.Fatalx(protocol+": serve", err) pkglog.Fatalx(protocol+": serve", err)
} }
servers = append(servers, serve) servers = append(servers, serve)
} }
@ -815,9 +816,9 @@ func Serve() {
SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}, SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256},
SupportedVersions: []uint16{tls.VersionTLS13}, SupportedVersions: []uint16{tls.VersionTLS13},
} }
xlog.Print("ensuring certificate availability", mlog.Field("hostname", host)) pkglog.Print("ensuring certificate availability", slog.Any("hostname", host))
if _, err := m.Manager.GetCertificate(hello); err != nil { if _, err := m.Manager.GetCertificate(hello); err != nil {
xlog.Errorx("requesting automatic certificate", err, mlog.Field("hostname", host)) pkglog.Errorx("requesting automatic certificate", err, slog.Any("hostname", host))
} }
} }
} }

34
http/webserver.go
View file

@ -25,6 +25,8 @@ import (
"syscall" "syscall"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/config" "github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
@ -149,8 +151,8 @@ table > tbody > tr:nth-child(odd) { background-color: #f8f8f8; }
// file is returned. Otherwise, for directories with ListFiles configured, a // file is returned. Otherwise, for directories with ListFiles configured, a
// directory listing is returned. // directory listing is returned.
func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *http.Request) (handled bool) { func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *http.Request) (handled bool) {
log := func() *mlog.Log { log := func() mlog.Log {
return xlog.WithContext(r.Context()) return pkglog.WithContext(r.Context())
} }
if r.Method != "GET" && r.Method != "HEAD" { if r.Method != "GET" && r.Method != "HEAD" {
if h.ContinueNotFound { if h.ContinueNotFound {
@ -217,7 +219,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
var ifi os.FileInfo var ifi os.FileInfo
ifi, err = index.Stat() ifi, err = index.Stat()
if err != nil { if err != nil {
log().Errorx("stat index.html in directory we cannot list", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) log().Errorx("stat index.html in directory we cannot list", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true return true
} }
@ -228,7 +230,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
http.Error(w, "403 - permission denied", http.StatusForbidden) http.Error(w, "403 - permission denied", http.StatusForbidden)
return true return true
} }
log().Errorx("open file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) log().Errorx("open file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true return true
} }
@ -236,7 +238,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
fi, err := f.Stat() fi, err := f.Stat()
if err != nil { if err != nil {
log().Errorx("stat file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) log().Errorx("stat file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true return true
} }
@ -274,7 +276,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
} }
} }
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
log().Errorx("stat for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) log().Errorx("stat for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true return true
} }
@ -315,7 +317,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
if err == io.EOF { if err == io.EOF {
break break
} else if err != nil { } else if err != nil {
log().Errorx("reading directory for file listing", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) log().Errorx("reading directory for file listing", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true return true
} }
@ -398,8 +400,8 @@ func HandleRedirect(h *config.WebRedirect, w http.ResponseWriter, r *http.Reques
// connections by monitoring the websocket handshake and then just passing along the // connections by monitoring the websocket handshake and then just passing along the
// websocket frames. // websocket frames.
func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) { func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) {
log := func() *mlog.Log { log := func() mlog.Log {
return xlog.WithContext(r.Context()) return pkglog.WithContext(r.Context())
} }
xr := *r xr := *r
@ -459,13 +461,13 @@ func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request,
// ReverseProxy will append any remaining path to the configured target URL. // ReverseProxy will append any remaining path to the configured target URL.
proxy := httputil.NewSingleHostReverseProxy(h.TargetURL) proxy := httputil.NewSingleHostReverseProxy(h.TargetURL)
proxy.FlushInterval = time.Duration(-1) // Flush after each write. proxy.FlushInterval = time.Duration(-1) // Flush after each write.
proxy.ErrorLog = golog.New(mlog.ErrWriter(mlog.New("net/http/httputil").WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0) proxy.ErrorLog = golog.New(mlog.LogWriter(mlog.New("net/http/httputil", nil).WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0)
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
if errors.Is(err, context.Canceled) { if errors.Is(err, context.Canceled) {
log().Debugx("forwarding request to backend webserver", err, mlog.Field("url", r.URL)) log().Debugx("forwarding request to backend webserver", err, slog.Any("url", r.URL))
return return
} }
log().Errorx("forwarding request to backend webserver", err, mlog.Field("url", r.URL)) log().Errorx("forwarding request to backend webserver", err, slog.Any("url", r.URL))
if os.IsTimeout(err) { if os.IsTimeout(err) {
http.Error(w, "504 - gateway timeout"+recvid(r), http.StatusGatewayTimeout) http.Error(w, "504 - gateway timeout"+recvid(r), http.StatusGatewayTimeout)
} else { } else {
@ -493,8 +495,8 @@ var errNotImplemented = errors.New("functionality not yet implemented")
// work for little benefit. Besides, the whole point of websockets is to exchange // work for little benefit. Besides, the whole point of websockets is to exchange
// bytes without HTTP being in the way, so let's do that. // bytes without HTTP being in the way, so let's do that.
func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) { func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) {
log := func() *mlog.Log { log := func() mlog.Log {
return xlog.WithContext(r.Context()) return pkglog.WithContext(r.Context())
} }
lw := w.(*loggingWriter) lw := w.(*loggingWriter)
@ -658,8 +660,8 @@ func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Reque
} }
func websocketTransact(ctx context.Context, targetURL *url.URL, r *http.Request) (rresp *http.Response, rconn net.Conn, rerr error) { func websocketTransact(ctx context.Context, targetURL *url.URL, r *http.Request) (rresp *http.Response, rconn net.Conn, rerr error) {
log := func() *mlog.Log { log := func() mlog.Log {
return xlog.WithContext(r.Context()) return pkglog.WithContext(r.Context())
} }
// Dial the backend, possibly doing TLS. We assume the net/http DefaultTransport is // Dial the backend, possibly doing TLS. We assume the net/http DefaultTransport is

6
imapserver/fetch.go
View file

@ -12,11 +12,11 @@ import (
"strings" "strings"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
@ -233,7 +233,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
for _, uid := range uids { for _, uid := range uids {
cmd.uid = uid cmd.uid = uid
mlog.Field("processing uid", mlog.Field("uid", uid)) cmd.conn.log.Debug("processing uid", slog.Any("uid", uid))
cmd.process(atts) cmd.process(atts)
} }
@ -326,7 +326,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
cmd.expungeIssued = true cmd.expungeIssued = true
return return
} }
cmd.conn.log.Infox("processing fetch attribute", err, mlog.Field("uid", cmd.uid)) cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
xuserErrorf("processing fetch attribute: %v", err) xuserErrorf("processing fetch attribute: %v", err)
}() }()

6
imapserver/fuzz_test.go
View file

@ -12,6 +12,7 @@ import (
"time" "time"
"github.com/mjl-/mox/imapclient" "github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -58,17 +59,18 @@ func FuzzServer(f *testing.F) {
f.Add(tag + cmd) f.Add(tag + cmd)
} }
log := mlog.New("imapserver", nil)
mox.Context = ctxbg mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/imapserverfuzz/mox.conf") mox.ConfigStaticPath = filepath.FromSlash("../testdata/imapserverfuzz/mox.conf")
mox.MustLoadConfig(true, false) mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
os.RemoveAll(dataDir) os.RemoveAll(dataDir)
acc, err := store.OpenAccount("mjl") acc, err := store.OpenAccount(log, "mjl")
if err != nil { if err != nil {
f.Fatalf("open account: %v", err) f.Fatalf("open account: %v", err)
} }
defer acc.Close() defer acc.Close()
err = acc.SetPassword("testtest") err = acc.SetPassword(log, "testtest")
if err != nil { if err != nil {
f.Fatalf("set password: %v", err) f.Fatalf("set password: %v", err)
} }

4
imapserver/parse.go
View file

@ -8,7 +8,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/mjl-/mox/mlog" "golang.org/x/exp/slog"
) )
var ( var (
@ -402,7 +402,7 @@ func (p *parser) xmailbox() string {
if !p.conn.enabled[capIMAP4rev2] { if !p.conn.enabled[capIMAP4rev2] {
ns, err := utf7decode(s) ns, err := utf7decode(s)
if err != nil { if err != nil {
p.conn.log.Infox("decoding utf7 or mailbox name", err, mlog.Field("name", s)) p.conn.log.Infox("decoding utf7 or mailbox name", err, slog.String("name", s))
} else { } else {
s = ns s = ns
} }

9
imapserver/search.go
View file

@ -5,10 +5,11 @@ import (
"net/textproto" "net/textproto"
"strings" "strings"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -393,7 +394,7 @@ func (s *search) match0(sk searchKey) bool {
lower := strings.ToLower(value) lower := strings.ToLower(value)
h, err := s.p.Header() h, err := s.p.Header()
if err != nil { if err != nil {
c.log.Debugx("parsing message header", err, mlog.Field("uid", s.uid)) c.log.Debugx("parsing message header", err, slog.Any("uid", s.uid))
return false return false
} }
for _, v := range h.Values(field) { for _, v := range h.Values(field) {
@ -517,7 +518,7 @@ func (s *search) match0(sk searchKey) bool {
} }
if s.p == nil { if s.p == nil {
c.log.Info("missing parsed message, not matching", mlog.Field("uid", s.uid)) c.log.Info("missing parsed message, not matching", slog.Any("uid", s.uid))
return false return false
} }
@ -546,7 +547,7 @@ func (s *search) match0(sk searchKey) bool {
lower := strings.ToLower(sk.astring) lower := strings.ToLower(sk.astring)
h, err := s.p.Header() h, err := s.p.Header()
if err != nil { if err != nil {
c.log.Errorx("parsing header for search", err, mlog.Field("uid", s.uid)) c.log.Errorx("parsing header for search", err, slog.Any("uid", s.uid))
return false return false
} }
k := textproto.CanonicalMIMEHeaderKey(sk.headerField) k := textproto.CanonicalMIMEHeaderKey(sk.headerField)

94
imapserver/server.go
View file

@ -56,10 +56,12 @@ import (
"runtime/debug" "runtime/debug"
"sort" "sort"
"strings" "strings"
"sync"
"time" "time"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -78,10 +80,6 @@ import (
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
// Most logging should be done through conn.log* functions.
// Only use imaplog in contexts without connection.
var xlog = mlog.New("imapserver")
var ( var (
metricIMAPConnection = promauto.NewCounterVec( metricIMAPConnection = promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
@ -180,7 +178,7 @@ type conn struct {
cmdMetric string // Currently executing, for metrics. cmdMetric string // Currently executing, for metrics.
cmdStart time.Time cmdStart time.Time
ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid. ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid.
log *mlog.Log log mlog.Log
enabled map[capability]bool // All upper-case. enabled map[capability]bool // All upper-case.
// Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with // Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with
@ -338,14 +336,15 @@ func Listen() {
var servers []func() var servers []func()
func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noRequireSTARTTLS bool) { func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noRequireSTARTTLS bool) {
log := mlog.New("imapserver", nil)
addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
if os.Getuid() == 0 { if os.Getuid() == 0 {
xlog.Print("listening for imap", mlog.Field("listener", listenerName), mlog.Field("addr", addr), mlog.Field("protocol", protocol)) log.Print("listening for imap", slog.String("listener", listenerName), slog.String("addr", addr), slog.String("protocol", protocol))
} }
network := mox.Network(ip) network := mox.Network(ip)
ln, err := mox.Listen(network, addr) ln, err := mox.Listen(network, addr)
if err != nil { if err != nil {
xlog.Fatalx("imap: listen for imap", err, mlog.Field("protocol", protocol), mlog.Field("listener", listenerName)) log.Fatalx("imap: listen for imap", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
} }
if xtls { if xtls {
ln = tls.NewListener(ln, tlsConfig) ln = tls.NewListener(ln, tlsConfig)
@ -355,7 +354,7 @@ func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config,
for { for {
conn, err := ln.Accept() conn, err := ln.Accept()
if err != nil { if err != nil {
xlog.Infox("imap: accept", err, mlog.Field("protocol", protocol), mlog.Field("listener", listenerName)) log.Infox("imap: accept", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
continue continue
} }
@ -441,7 +440,7 @@ func (c *conn) Write(buf []byte) (int, error) {
return n, nil return n, nil
} }
func (c *conn) xtrace(level mlog.Level) func() { func (c *conn) xtrace(level slog.Level) func() {
c.xflush() c.xflush()
c.tr.SetTrace(level) c.tr.SetTrace(level)
c.tw.SetTrace(level) c.tw.SetTrace(level)
@ -469,7 +468,7 @@ func (c *conn) readline0() (string, error) {
err := c.conn.SetReadDeadline(time.Now().Add(d)) err := c.conn.SetReadDeadline(time.Now().Add(d))
c.log.Check(err, "setting read deadline") c.log.Check(err, "setting read deadline")
line, err := bufpool.Readline(c.br) line, err := bufpool.Readline(c.log, c.br)
if err != nil && errors.Is(err, moxio.ErrLineTooLong) { if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
return "", fmt.Errorf("%s (%w)", err, errProtocol) return "", fmt.Errorf("%s (%w)", err, errProtocol)
} else if err != nil { } else if err != nil {
@ -629,15 +628,18 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
cmd: "(greeting)", cmd: "(greeting)",
cmdStart: time.Now(), cmdStart: time.Now(),
} }
c.log = xlog.MoreFields(func() []mlog.Pair { var logmutex sync.Mutex
c.log = mlog.New("imapserver", nil).WithFunc(func() []slog.Attr {
logmutex.Lock()
defer logmutex.Unlock()
now := time.Now() now := time.Now()
l := []mlog.Pair{ l := []slog.Attr{
mlog.Field("cid", c.cid), slog.Int64("cid", c.cid),
mlog.Field("delta", now.Sub(c.lastlog)), slog.Duration("delta", now.Sub(c.lastlog)),
} }
c.lastlog = now c.lastlog = now
if c.username != "" { if c.username != "" {
l = append(l, mlog.Field("username", c.username)) l = append(l, slog.String("username", c.username))
} }
return l return l
}) })
@ -662,7 +664,7 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
} }
} }
c.log.Info("new connection", mlog.Field("remote", c.conn.RemoteAddr()), mlog.Field("local", c.conn.LocalAddr()), mlog.Field("tls", xtls), mlog.Field("listener", listenerName)) c.log.Info("new connection", slog.Any("remote", c.conn.RemoteAddr()), slog.Any("local", c.conn.LocalAddr()), slog.Bool("tls", xtls), slog.String("listener", listenerName))
defer func() { defer func() {
c.conn.Close() c.conn.Close()
@ -681,7 +683,7 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
} else if err, ok := x.(error); ok && isClosed(err) { } else if err, ok := x.(error); ok && isClosed(err) {
c.log.Infox("connection closed", err) c.log.Infox("connection closed", err)
} else { } else {
c.log.Error("unhandled panic", mlog.Field("err", x)) c.log.Error("unhandled panic", slog.Any("err", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Imapserver) metrics.PanicInc(metrics.Imapserver)
} }
@ -703,13 +705,13 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
// If remote IP/network resulted in too many authentication failures, refuse to serve. // If remote IP/network resulted in too many authentication failures, refuse to serve.
if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) { if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
metrics.AuthenticationRatelimitedInc("imap") metrics.AuthenticationRatelimitedInc("imap")
c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP)) c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
c.writelinef("* BYE too many auth failures") c.writelinef("* BYE too many auth failures")
return return
} }
if !limiterConnections.Add(c.remoteIP, time.Now(), 1) { if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP)) c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
c.writelinef("* BYE too many open connections from your ip or network") c.writelinef("* BYE too many open connections from your ip or network")
return return
} }
@ -744,9 +746,9 @@ func (c *conn) command() {
metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
}() }()
logFields := []mlog.Pair{ logFields := []slog.Attr{
mlog.Field("cmd", c.cmd), slog.String("cmd", c.cmd),
mlog.Field("duration", time.Since(c.cmdStart)), slog.Duration("duration", time.Since(c.cmdStart)),
} }
c.cmd = "" c.cmd = ""
@ -761,7 +763,7 @@ func (c *conn) command() {
} }
err, ok := x.(error) err, ok := x.(error)
if !ok { if !ok {
c.log.Error("imap command panic", append([]mlog.Pair{mlog.Field("panic", x)}, logFields...)...) c.log.Error("imap command panic", append([]slog.Attr{slog.Any("panic", x)}, logFields...)...)
result = "panic" result = "panic"
panic(x) panic(x)
} }
@ -786,7 +788,7 @@ func (c *conn) command() {
panic(errIO) panic(errIO)
} }
c.log.Debugx("imap command syntax error", sxerr.err, logFields...) c.log.Debugx("imap command syntax error", sxerr.err, logFields...)
c.log.Info("imap syntax error", mlog.Field("lastline", c.lastLine)) c.log.Info("imap syntax error", slog.String("lastline", c.lastLine))
fatal := strings.HasSuffix(c.lastLine, "+}") fatal := strings.HasSuffix(c.lastLine, "+}")
if fatal { if fatal {
err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
@ -865,7 +867,7 @@ func (c *conn) broadcast(changes []store.Change) {
if len(changes) == 0 { if len(changes) == 0 {
return return
} }
c.log.Debug("broadcast changes", mlog.Field("changes", changes)) c.log.Debug("broadcast changes", slog.Any("changes", changes))
c.comm.Broadcast(changes) c.comm.Broadcast(changes)
} }
@ -1184,7 +1186,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute)) err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
c.log.Check(err, "setting write deadline") c.log.Check(err, "setting write deadline")
c.log.Debug("applying changes", mlog.Field("changes", changes)) c.log.Debug("applying changes", slog.Any("changes", changes))
// Only keep changes for the selected mailbox, and changes that are always relevant. // Only keep changes for the selected mailbox, and changes that are always relevant.
var n []store.Change var n []store.Change
@ -1404,7 +1406,7 @@ func (c *conn) cmdID(tag, cmd string, p *parser) {
p.xempty() p.xempty()
// We just log the client id. // We just log the client id.
c.log.Info("client id", mlog.Field("params", params)) c.log.Info("client id", slog.Any("params", params))
// Response syntax: ../rfc/2971:243 // Response syntax: ../rfc/2971:243
// We send our name and version. ../rfc/2971:193 // We send our name and version. ../rfc/2971:193
@ -1448,7 +1450,7 @@ func (c *conn) cmdStarttls(tag, cmd string, p *parser) {
} }
cancel() cancel()
tlsversion, ciphersuite := mox.TLSInfo(tlsConn) tlsversion, ciphersuite := mox.TLSInfo(tlsConn)
c.log.Debug("tls server handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite)) c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite))
c.conn = tlsConn c.conn = tlsConn
c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn) c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
@ -1560,11 +1562,11 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role") xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role")
} }
acc, err := store.OpenEmailAuth(authc, password) acc, err := store.OpenEmailAuth(c.log, authc, password)
if err != nil { if err != nil {
if errors.Is(err, store.ErrUnknownCredentials) { if errors.Is(err, store.ErrUnknownCredentials) {
authResult = "badcreds" authResult = "badcreds"
c.log.Info("authentication failed", mlog.Field("username", authc)) c.log.Info("authentication failed", slog.String("username", authc))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
} }
xusercodeErrorf("", "error") xusercodeErrorf("", "error")
@ -1588,11 +1590,11 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xsyntaxErrorf("malformed cram-md5 response") xsyntaxErrorf("malformed cram-md5 response")
} }
addr := t[0] addr := t[0]
c.log.Debug("cram-md5 auth", mlog.Field("address", addr)) c.log.Debug("cram-md5 auth", slog.String("address", addr))
acc, _, err := store.OpenEmail(addr) acc, _, err := store.OpenEmail(c.log, addr)
if err != nil { if err != nil {
if errors.Is(err, store.ErrUnknownCredentials) { if errors.Is(err, store.ErrUnknownCredentials) {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
} }
xserverErrorf("looking up address: %v", err) xserverErrorf("looking up address: %v", err)
@ -1608,7 +1610,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error { err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
password, err := bstore.QueryTx[store.Password](tx).Get() password, err := bstore.QueryTx[store.Password](tx).Get()
if err == bstore.ErrAbsent { if err == bstore.ErrAbsent {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
} }
if err != nil { if err != nil {
@ -1622,8 +1624,8 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xcheckf(err, "tx read") xcheckf(err, "tx read")
}) })
if ipadhash == nil || opadhash == nil { if ipadhash == nil || opadhash == nil {
c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", mlog.Field("username", addr)) c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", addr))
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
} }
@ -1632,7 +1634,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
opadhash.Write(ipadhash.Sum(nil)) opadhash.Write(ipadhash.Sum(nil))
digest := fmt.Sprintf("%x", opadhash.Sum(nil)) digest := fmt.Sprintf("%x", opadhash.Sum(nil))
if digest != t[1] { if digest != t[1] {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
} }
@ -1659,8 +1661,8 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
if err != nil { if err != nil {
xsyntaxErrorf("starting scram: %s", err) xsyntaxErrorf("starting scram: %s", err)
} }
c.log.Debug("scram auth", mlog.Field("authentication", ss.Authentication)) c.log.Debug("scram auth", slog.String("authentication", ss.Authentication))
acc, _, err := store.OpenEmail(ss.Authentication) acc, _, err := store.OpenEmail(c.log, ss.Authentication)
if err != nil { if err != nil {
// todo: we could continue scram with a generated salt, deterministically generated // todo: we could continue scram with a generated salt, deterministically generated
// from the username. that way we don't have to store anything but attackers cannot // from the username. that way we don't have to store anything but attackers cannot
@ -1686,7 +1688,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xscram = password.SCRAMSHA256 xscram = password.SCRAMSHA256
} }
if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) { if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) {
c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", mlog.Field("address", ss.Authentication)) c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("address", ss.Authentication))
xuserErrorf("scram not possible") xuserErrorf("scram not possible")
} }
xcheckf(err, "fetching credentials") xcheckf(err, "fetching credentials")
@ -1706,7 +1708,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
c.readline(false) // Should be "*" for cancellation. c.readline(false) // Should be "*" for cancellation.
if errors.Is(err, scram.ErrInvalidProof) { if errors.Is(err, scram.ErrInvalidProof) {
authResult = "badcreds" authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
} }
xuserErrorf("server final: %w", err) xuserErrorf("server final: %w", err)
@ -1770,13 +1772,13 @@ func (c *conn) cmdLogin(tag, cmd string, p *parser) {
} }
}() }()
acc, err := store.OpenEmailAuth(userid, password) acc, err := store.OpenEmailAuth(c.log, userid, password)
if err != nil { if err != nil {
authResult = "badcreds" authResult = "badcreds"
var code string var code string
if errors.Is(err, store.ErrUnknownCredentials) { if errors.Is(err, store.ErrUnknownCredentials) {
code = "AUTHENTICATIONFAILED" code = "AUTHENTICATIONFAILED"
c.log.Info("failed authentication attempt", mlog.Field("username", userid), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", userid), slog.Any("remote", c.remoteIP))
} }
xusercodeErrorf(code, "login failed") xusercodeErrorf(code, "login failed")
} }
@ -2251,7 +2253,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
for _, mID := range removeMessageIDs { for _, mID := range removeMessageIDs {
p := c.account.MessagePath(mID) p := c.account.MessagePath(mID)
err := os.Remove(p) err := os.Remove(p)
c.log.Check(err, "removing message file for mailbox delete", mlog.Field("path", p)) c.log.Check(err, "removing message file for mailbox delete", slog.String("path", p))
} }
c.ok(tag, cmd) c.ok(tag, cmd)
@ -2674,7 +2676,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
} }
// Read the message into a temporary file. // Read the message into a temporary file.
msgFile, err := store.CreateMessageTemp("imap-append") msgFile, err := store.CreateMessageTemp(c.log, "imap-append")
xcheckf(err, "creating temp file for message") xcheckf(err, "creating temp file for message")
defer func() { defer func() {
p := msgFile.Name() p := msgFile.Name()
@ -3273,7 +3275,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
} }
for dir := range syncDirs { for dir := range syncDirs {
err := moxio.SyncDir(dir) err := moxio.SyncDir(c.log, dir)
xcheckf(err, "sync directory") xcheckf(err, "sync directory")
} }

6
imapserver/server_test.go
View file

@ -17,12 +17,14 @@ import (
"time" "time"
"github.com/mjl-/mox/imapclient" "github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
var ctxbg = context.Background() var ctxbg = context.Background()
var pkglog = mlog.New("imapserver", nil)
func init() { func init() {
sanityChecks = true sanityChecks = true
@ -341,10 +343,10 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
mox.Context = ctxbg mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf") mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf")
mox.MustLoadConfig(true, false) mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl") acc, err := store.OpenAccount(pkglog, "mjl")
tcheck(t, err, "open account") tcheck(t, err, "open account")
if first { if first {
err = acc.SetPassword("testtest") err = acc.SetPassword(pkglog, "testtest")
tcheck(t, err, "set password") tcheck(t, err, "set password")
} }
switchStop := func() {} switchStop := func() {}

29
import.go
View file

@ -16,11 +16,11 @@ import (
"time" "time"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/config" "github.com/mjl-/mox/config"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/metrics" "github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -119,10 +119,9 @@ func xcmdXImport(mbox bool, c *cmd) {
} }
defer store.Switchboard()() defer store.Switchboard()()
xlog := mlog.New("import")
cconn, sconn := net.Pipe() cconn, sconn := net.Pipe()
clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: xlog} clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: c.log}
serverctl := ctl{conn: sconn, r: bufio.NewReader(sconn), log: xlog} serverctl := ctl{conn: sconn, r: bufio.NewReader(sconn), log: c.log}
go servectlcmd(context.Background(), &serverctl, func() {}) go servectlcmd(context.Background(), &serverctl, func() {})
ctlcmdImport(&clientctl, mbox, account, args[1], args[2]) ctlcmdImport(&clientctl, mbox, account, args[1], args[2])
@ -177,7 +176,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
if mbox { if mbox {
kind = "mbox" kind = "mbox"
} }
ctl.log.Info("importing messages", mlog.Field("kind", kind), mlog.Field("account", account), mlog.Field("mailbox", mailbox), mlog.Field("source", src)) ctl.log.Info("importing messages", slog.String("kind", kind), slog.String("account", account), slog.String("mailbox", mailbox), slog.String("source", src))
var err error var err error
var mboxf *os.File var mboxf *os.File
@ -186,7 +185,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
// Open account, creating a database file if it doesn't exist yet. It must be known // Open account, creating a database file if it doesn't exist yet. It must be known
// in the configuration file. // in the configuration file.
a, err := store.OpenAccount(account) a, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "opening account") ctl.xcheck(err, "opening account")
defer func() { defer func() {
if a != nil { if a != nil {
@ -222,13 +221,13 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
if mbox { if mbox {
mboxf, err = os.Open(src) mboxf, err = os.Open(src)
ctl.xcheck(err, "open mbox file") ctl.xcheck(err, "open mbox file")
msgreader = store.NewMboxReader(store.CreateMessageTemp, src, mboxf, ctl.log) msgreader = store.NewMboxReader(ctl.log, store.CreateMessageTemp, src, mboxf)
} else { } else {
mdnewf, err = os.Open(filepath.Join(src, "new")) mdnewf, err = os.Open(filepath.Join(src, "new"))
ctl.xcheck(err, "open subdir new of maildir") ctl.xcheck(err, "open subdir new of maildir")
mdcurf, err = os.Open(filepath.Join(src, "cur")) mdcurf, err = os.Open(filepath.Join(src, "cur"))
ctl.xcheck(err, "open subdir cur of maildir") ctl.xcheck(err, "open subdir cur of maildir")
msgreader = store.NewMaildirReader(store.CreateMessageTemp, mdnewf, mdcurf, ctl.log) msgreader = store.NewMaildirReader(ctl.log, store.CreateMessageTemp, mdnewf, mdcurf)
} }
tx, err := a.DB.Begin(ctx, true) tx, err := a.DB.Begin(ctx, true)
@ -253,7 +252,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
} }
if x != ctl.x { if x != ctl.x {
ctl.log.Error("import error", mlog.Field("panic", fmt.Errorf("%v", x))) ctl.log.Error("import error", slog.String("panic", fmt.Sprintf("%v", x)))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Import) metrics.PanicInc(metrics.Import)
} else { } else {
@ -263,7 +262,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
for _, id := range deliveredIDs { for _, id := range deliveredIDs {
p := a.MessagePath(id) p := a.MessagePath(id)
err := os.Remove(p) err := os.Remove(p)
ctl.log.Check(err, "closing message file after import error", mlog.Field("path", p)) ctl.log.Check(err, "closing message file after import error", slog.String("path", p))
} }
ctl.xerror(fmt.Sprintf("import error: %v", x)) ctl.xerror(fmt.Sprintf("import error: %v", x))
@ -282,7 +281,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
err := a.DeliverMessage(ctl.log, tx, m, mf, sync, notrain, nothreads) err := a.DeliverMessage(ctl.log, tx, m, mf, sync, notrain, nothreads)
ctl.xcheck(err, "delivering message") ctl.xcheck(err, "delivering message")
deliveredIDs = append(deliveredIDs, m.ID) deliveredIDs = append(deliveredIDs, m.ID)
ctl.log.Debug("delivered message", mlog.Field("id", m.ID)) ctl.log.Debug("delivered message", slog.Int64("id", m.ID))
changes = append(changes, m.ChangeAddUID()) changes = append(changes, m.ChangeAddUID())
} }
@ -319,9 +318,9 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
mb.Add(m.MailboxCounts()) mb.Add(m.MailboxCounts())
// Parse message and store parsed information for later fast retrieval. // Parse message and store parsed information for later fast retrieval.
p, err := message.EnsurePart(ctl.log, false, msgf, m.Size) p, err := message.EnsurePart(ctl.log.Logger, false, msgf, m.Size)
if err != nil { if err != nil {
ctl.log.Infox("parsing message, continuing", err, mlog.Field("path", origPath)) ctl.log.Infox("parsing message, continuing", err, slog.String("path", origPath))
} }
m.ParsedBuf, err = json.Marshal(p) m.ParsedBuf, err = json.Marshal(p)
ctl.xcheck(err, "marshal parsed message structure") ctl.xcheck(err, "marshal parsed message structure")
@ -345,7 +344,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
m.JunkFlagsForMailbox(mb, conf) m.JunkFlagsForMailbox(mb, conf)
if jf != nil && m.NeedsTraining() { if jf != nil && m.NeedsTraining() {
if words, err := jf.ParseMessage(p); err != nil { if words, err := jf.ParseMessage(p); err != nil {
ctl.log.Infox("parsing message for updating junk filter", err, mlog.Field("parse", ""), mlog.Field("path", origPath)) ctl.log.Infox("parsing message for updating junk filter", err, slog.String("parse", ""), slog.String("path", origPath))
} else { } else {
err = jf.Train(ctx, !m.Junk, words) err = jf.Train(ctx, !m.Junk, words)
ctl.xcheck(err, "training junk filter") ctl.xcheck(err, "training junk filter")
@ -407,7 +406,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
err = tx.Commit() err = tx.Commit()
ctl.xcheck(err, "commit") ctl.xcheck(err, "commit")
tx = nil tx = nil
ctl.log.Info("delivered messages through import", mlog.Field("count", len(deliveredIDs))) ctl.log.Info("delivered messages through import", slog.Int("count", len(deliveredIDs)))
deliveredIDs = nil deliveredIDs = nil
store.BroadcastChanges(a, changes) store.BroadcastChanges(a, changes)

28
integration_test.go
View file

@ -14,6 +14,8 @@ import (
"testing" "testing"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/imapclient" "github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
@ -30,7 +32,7 @@ func tcheck(t *testing.T, err error, errmsg string) {
} }
func TestDeliver(t *testing.T) { func TestDeliver(t *testing.T) {
xlog := mlog.New("integration") log := mlog.New("integration", nil)
mlog.Logfmt = true mlog.Logfmt = true
hostname, err := os.Hostname() hostname, err := os.Hostname()
@ -129,7 +131,7 @@ This is the message.
`, mailfrom, rcptto) `, mailfrom, rcptto)
msg = strings.ReplaceAll(msg, "\n", "\r\n") msg = strings.ReplaceAll(msg, "\n", "\r\n")
auth := []sasl.Client{sasl.NewClientPlain(mailfrom, password)} auth := []sasl.Client{sasl.NewClientPlain(mailfrom, password)}
c, err := smtpclient.New(mox.Context, xlog, conn, smtpclient.TLSSkip, false, ourHostname, dns.Domain{ASCII: desthost}, smtpclient.Opts{Auth: auth}) c, err := smtpclient.New(mox.Context, log.Logger, conn, smtpclient.TLSSkip, false, ourHostname, dns.Domain{ASCII: desthost}, smtpclient.Opts{Auth: auth})
tcheck(t, err, "smtp hello") tcheck(t, err, "smtp hello")
err = c.Deliver(mox.Context, mailfrom, rcptto, int64(len(msg)), strings.NewReader(msg), false, false, false) err = c.Deliver(mox.Context, mailfrom, rcptto, int64(len(msg)), strings.NewReader(msg), false, false, false)
tcheck(t, err, "deliver with smtp") tcheck(t, err, "deliver with smtp")
@ -142,35 +144,35 @@ This is the message.
tcheck(t, err, "dial submission") tcheck(t, err, "dial submission")
defer conn.Close() defer conn.Close()
xlog.Print("submitting email to moxacmepebble, waiting for imap notification at moxmail2") log.Print("submitting email to moxacmepebble, waiting for imap notification at moxmail2")
t0 := time.Now() t0 := time.Now()
deliver(true, true, "moxmail2.mox2.example:993", "moxtest2@mox2.example", "accountpass4321", func() { deliver(true, true, "moxmail2.mox2.example:993", "moxtest2@mox2.example", "accountpass4321", func() {
submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "moxtest2@mox2.example") submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "moxtest2@mox2.example")
}) })
xlog.Print("success", mlog.Field("duration", time.Since(t0))) log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to moxmail2, waiting for imap notification at moxacmepebble") log.Print("submitting email to moxmail2, waiting for imap notification at moxacmepebble")
t0 = time.Now() t0 = time.Now()
deliver(true, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() { deliver(true, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() {
submit(true, "moxtest2@mox2.example", "accountpass4321", "moxmail2.mox2.example:465", "moxtest1@mox1.example") submit(true, "moxtest2@mox2.example", "accountpass4321", "moxmail2.mox2.example:465", "moxtest1@mox1.example")
}) })
xlog.Print("success", mlog.Field("duration", time.Since(t0))) log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to postfix, waiting for imap notification at moxacmepebble") log.Print("submitting email to postfix, waiting for imap notification at moxacmepebble")
t0 = time.Now() t0 = time.Now()
deliver(false, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() { deliver(false, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() {
submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "root@postfix.example") submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "root@postfix.example")
}) })
xlog.Print("success", mlog.Field("duration", time.Since(t0))) log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to localserve") log.Print("submitting email to localserve")
t0 = time.Now() t0 = time.Now()
deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() { deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() {
submit(false, "mox@localhost", "moxmoxmox", "localserve.mox1.example:1587", "moxtest1@mox1.example") submit(false, "mox@localhost", "moxmoxmox", "localserve.mox1.example:1587", "moxtest1@mox1.example")
}) })
xlog.Print("success", mlog.Field("duration", time.Since(t0))) log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to localserve") log.Print("submitting email to localserve")
t0 = time.Now() t0 = time.Now()
deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() { deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() {
cmd := exec.Command("go", "run", ".", "sendmail", "mox@localhost") cmd := exec.Command("go", "run", ".", "sendmail", "mox@localhost")
@ -182,8 +184,8 @@ a message.
var out strings.Builder var out strings.Builder
cmd.Stdout = &out cmd.Stdout = &out
err := cmd.Run() err := cmd.Run()
xlog.Print("sendmail", mlog.Field("output", out.String())) log.Print("sendmail", slog.String("output", out.String()))
tcheck(t, err, "sendmail") tcheck(t, err, "sendmail")
}) })
xlog.Print("success", mlog.Field("duration", time.Since(t0))) log.Print("success", slog.Any("duration", time.Since(t0)))
} }

6
iprev/iprev.go
View file

@ -9,6 +9,8 @@ import (
"net" "net"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -16,7 +18,7 @@ import (
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
var xlog = mlog.New("iprev") var xlog = mlog.New("iprev", nil)
var ( var (
metricIPRev = promauto.NewHistogramVec( metricIPRev = promauto.NewHistogramVec(
@ -61,7 +63,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, ip net.IP) (rstatus Stat
start := time.Now() start := time.Now()
defer func() { defer func() {
metricIPRev.WithLabelValues(string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second)) metricIPRev.WithLabelValues(string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("iprev lookup result", rerr, mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("duration", time.Since(start))) log.Debugx("iprev lookup result", rerr, slog.Any("ip", ip), slog.Any("status", rstatus), slog.Duration("duration", time.Since(start)))
}() }()
revNames, result, revErr := dns.WithPackage(resolver, "iprev").LookupAddr(ctx, ip.String()) revNames, result, revErr := dns.WithPackage(resolver, "iprev").LookupAddr(ctx, ip.String())

16
junk.go
View file

@ -93,7 +93,7 @@ func cmdJunkTrain(c *cmd) {
} }
a.SetLogLevel() a.SetLogLevel()
f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath)) f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath))
defer func() { defer func() {
if err := f.Close(); err != nil { if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err) log.Printf("closing junk filter: %v", err)
@ -122,7 +122,7 @@ func cmdJunkCheck(c *cmd) {
} }
a.SetLogLevel() a.SetLogLevel()
f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false)) f := must(junk.OpenFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath, false))
defer func() { defer func() {
if err := f.Close(); err != nil { if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err) log.Printf("closing junk filter: %v", err)
@ -146,7 +146,7 @@ func cmdJunkTest(c *cmd) {
} }
a.SetLogLevel() a.SetLogLevel()
f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false)) f := must(junk.OpenFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath, false))
defer func() { defer func() {
if err := f.Close(); err != nil { if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err) log.Printf("closing junk filter: %v", err)
@ -202,7 +202,7 @@ messages are shuffled, with optional random seed.`
} }
a.SetLogLevel() a.SetLogLevel()
f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath)) f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath))
defer func() { defer func() {
if err := f.Close(); err != nil { if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err) log.Printf("closing junk filter: %v", err)
@ -293,7 +293,7 @@ func cmdJunkPlay(c *cmd) {
} }
a.SetLogLevel() a.SetLogLevel()
f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath)) f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath))
defer func() { defer func() {
if err := f.Close(); err != nil { if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err) log.Printf("closing junk filter: %v", err)
@ -310,8 +310,6 @@ func cmdJunkPlay(c *cmd) {
var nbad, nnodate, nham, nspam, nsent int var nbad, nnodate, nham, nspam, nsent int
jlog := mlog.New("junkplay")
scanDir := func(dir string, ham, sent bool) { scanDir := func(dir string, ham, sent bool) {
for _, name := range listDir(dir) { for _, name := range listDir(dir) {
path := filepath.Join(dir, name) path := filepath.Join(dir, name)
@ -319,7 +317,7 @@ func cmdJunkPlay(c *cmd) {
xcheckf(err, "open %q", path) xcheckf(err, "open %q", path)
fi, err := mf.Stat() fi, err := mf.Stat()
xcheckf(err, "stat %q", path) xcheckf(err, "stat %q", path)
p, err := message.EnsurePart(jlog, false, mf, fi.Size()) p, err := message.EnsurePart(c.log.Logger, false, mf, fi.Size())
if err != nil { if err != nil {
nbad++ nbad++
if err := mf.Close(); err != nil { if err := mf.Close(); err != nil {
@ -399,7 +397,7 @@ func cmdJunkPlay(c *cmd) {
}() }()
fi, err := mf.Stat() fi, err := mf.Stat()
xcheckf(err, "stat %q", path) xcheckf(err, "stat %q", path)
p, err := message.EnsurePart(jlog, false, mf, fi.Size()) p, err := message.EnsurePart(c.log.Logger, false, mf, fi.Size())
if err != nil { if err != nil {
log.Printf("bad sent message %q: %s", path, err) log.Printf("bad sent message %q: %s", path, err)
return return

61
junk/filter.go
View file

@ -21,6 +21,8 @@ import (
"sort" "sort"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
@ -28,8 +30,6 @@ import (
) )
var ( var (
xlog = mlog.New("junk")
// errBadContentType = errors.New("bad content-type") // sure sign of spam, todo: use this error // errBadContentType = errors.New("bad content-type") // sure sign of spam, todo: use this error
errClosed = errors.New("filter is closed") errClosed = errors.New("filter is closed")
) )
@ -62,7 +62,7 @@ var DBTypes = []any{wordscore{}} // Stored in DB.
type Filter struct { type Filter struct {
Params Params
log *mlog.Log // For logging cid. log mlog.Log // For logging cid.
closed bool closed bool
modified bool // Whether any modifications are pending. Cleared by Save. modified bool // Whether any modifications are pending. Cleared by Save.
hams, spams uint32 // Message count, stored in db under word "-". hams, spams uint32 // Message count, stored in db under word "-".
@ -112,7 +112,7 @@ func (f *Filter) Close() error {
return err return err
} }
func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) { func OpenFilter(ctx context.Context, log mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) {
var bloom *Bloom var bloom *Bloom
if loadBloom { if loadBloom {
var err error var err error
@ -160,7 +160,7 @@ func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloom
// filter is marked as new until the first save, will be done automatically if // filter is marked as new until the first save, will be done automatically if
// TrainDirs is called. If the bloom and/or database files exist, an error is // TrainDirs is called. If the bloom and/or database files exist, an error is
// returned. // returned.
func NewFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) { func NewFilter(ctx context.Context, log mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) {
var err error var err error
if _, err := os.Stat(bloomPath); err == nil { if _, err := os.Stat(bloomPath); err == nil {
return nil, fmt.Errorf("bloom filter already exists on disk: %s", bloomPath) return nil, fmt.Errorf("bloom filter already exists on disk: %s", bloomPath)
@ -220,7 +220,7 @@ func openBloom(path string) (*Bloom, error) {
return NewBloom(buf, bloomK) return NewBloom(buf, bloomK)
} }
func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr error) { func newDB(ctx context.Context, log mlog.Log, path string) (db *bstore.DB, rerr error) {
// Remove any existing files. // Remove any existing files.
os.Remove(path) os.Remove(path)
@ -272,7 +272,7 @@ func (f *Filter) Save() error {
return words[i] < words[j] return words[i] < words[j]
}) })
f.log.Debug("inserting words in junkfilter db", mlog.Field("words", len(f.changed))) f.log.Debug("inserting words in junkfilter db", slog.Any("words", len(f.changed)))
// start := time.Now() // start := time.Now()
if f.isNew { if f.isNew {
if err := f.db.HintAppend(true, wordscore{}); err != nil { if err := f.db.HintAppend(true, wordscore{}); err != nil {
@ -318,7 +318,7 @@ func (f *Filter) Save() error {
f.changed = map[string]word{} f.changed = map[string]word{}
f.modified = false f.modified = false
f.isNew = false f.isNew = false
// f.log.Info("wrote filter to db", mlog.Field("duration", time.Since(start))) // f.log.Info("wrote filter to db", slog.Any("duration", time.Since(start)))
return nil return nil
} }
@ -378,7 +378,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (
expect[w] = struct{}{} expect[w] = struct{}{}
} }
if len(unknowns) > 0 { if len(unknowns) > 0 {
f.log.Debug("unknown words in bloom filter, showing max 50", mlog.Field("words", unknowns), mlog.Field("totalunknown", totalUnknown), mlog.Field("totalwords", len(words))) f.log.Debug("unknown words in bloom filter, showing max 50", slog.Any("words", unknowns), slog.Any("totalunknown", totalUnknown), slog.Any("totalwords", len(words)))
} }
// Fetch words from database. // Fetch words from database.
@ -391,7 +391,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (
delete(expect, w) delete(expect, w)
f.cache[w] = c f.cache[w] = c
} }
f.log.Debug("unknown words in db", mlog.Field("words", expect), mlog.Field("totalunknown", len(expect)), mlog.Field("totalwords", len(words))) f.log.Debug("unknown words in db", slog.Any("words", expect), slog.Any("totalunknown", len(expect)), slog.Any("totalwords", len(words)))
} }
for w := range words { for w := range words {
@ -474,7 +474,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (
eta += math.Log(1-x.R) - math.Log(x.R) eta += math.Log(1-x.R) - math.Log(x.R)
} }
f.log.Debug("top words", mlog.Field("hams", topHam), mlog.Field("spams", topSpam)) f.log.Debug("top words", slog.Any("hams", topHam), slog.Any("spams", topSpam))
prob := 1 / (1 + math.Pow(math.E, eta)) prob := 1 / (1 + math.Pow(math.E, eta))
return prob, len(topHam), len(topSpam), nil return prob, len(topHam), len(topSpam), nil
@ -502,7 +502,7 @@ func (f *Filter) ClassifyMessagePath(ctx context.Context, path string) (probabil
} }
func (f *Filter) ClassifyMessageReader(ctx context.Context, mf io.ReaderAt, size int64) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) { func (f *Filter) ClassifyMessageReader(ctx context.Context, mf io.ReaderAt, size int64) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
m, err := message.EnsurePart(f.log, false, mf, size) m, err := message.EnsurePart(f.log.Logger, false, mf, size)
if err != nil && errors.Is(err, message.ErrBadContentType) { if err != nil && errors.Is(err, message.ErrBadContentType) {
// Invalid content-type header is a sure sign of spam. // Invalid content-type header is a sure sign of spam.
//f.log.Infox("parsing content", err) //f.log.Infox("parsing content", err)
@ -568,7 +568,7 @@ func (f *Filter) Train(ctx context.Context, ham bool, words map[string]struct{})
} }
func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error { func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error {
p, _ := message.EnsurePart(f.log, false, r, size) p, _ := message.EnsurePart(f.log.Logger, false, r, size)
words, err := f.ParseMessage(p) words, err := f.ParseMessage(p)
if err != nil { if err != nil {
return fmt.Errorf("parsing mail contents: %v", err) return fmt.Errorf("parsing mail contents: %v", err)
@ -577,7 +577,7 @@ func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ha
} }
func (f *Filter) UntrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error { func (f *Filter) UntrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error {
p, _ := message.EnsurePart(f.log, false, r, size) p, _ := message.EnsurePart(f.log.Logger, false, r, size)
words, err := f.ParseMessage(p) words, err := f.ParseMessage(p)
if err != nil { if err != nil {
return fmt.Errorf("parsing mail contents: %v", err) return fmt.Errorf("parsing mail contents: %v", err)
@ -648,7 +648,7 @@ func (f *Filter) TrainDir(dir string, files []string, ham bool) (n, malformed ui
p := filepath.Join(dir, name) p := filepath.Join(dir, name)
valid, words, err := f.tokenizeMail(p) valid, words, err := f.tokenizeMail(p)
if err != nil { if err != nil {
// f.log.Infox("tokenizing mail", err, mlog.Field("path", p)) // f.log.Infox("tokenizing mail", err, slog.Any("path", p))
malformed++ malformed++
continue continue
} }
@ -720,21 +720,20 @@ func (f *Filter) TrainDirs(hamDir, sentDir, spamDir string, hamFiles, sentFiles,
dbSize := f.fileSize(f.dbPath) dbSize := f.fileSize(f.dbPath)
bloomSize := f.fileSize(f.bloomPath) bloomSize := f.fileSize(f.bloomPath)
fields := []mlog.Pair{ f.log.Print("training done",
mlog.Field("hams", hams), slog.Any("hams", hams),
mlog.Field("hamtime", tham), slog.Any("hamtime", tham),
mlog.Field("hammalformed", hamMalformed), slog.Any("hammalformed", hamMalformed),
mlog.Field("sent", sent), slog.Any("sent", sent),
mlog.Field("senttime", tsent), slog.Any("senttime", tsent),
mlog.Field("sentmalformed", sentMalformed), slog.Any("sentmalformed", sentMalformed),
mlog.Field("spams", f.spams), slog.Any("spams", f.spams),
mlog.Field("spamtime", tspam), slog.Any("spamtime", tspam),
mlog.Field("spammalformed", spamMalformed), slog.Any("spammalformed", spamMalformed),
mlog.Field("dbsize", fmt.Sprintf("%.1fmb", float64(dbSize)/(1024*1024))), slog.Any("dbsize", fmt.Sprintf("%.1fmb", float64(dbSize)/(1024*1024))),
mlog.Field("bloomsize", fmt.Sprintf("%.1fmb", float64(bloomSize)/(1024*1024))), slog.Any("bloomsize", fmt.Sprintf("%.1fmb", float64(bloomSize)/(1024*1024))),
mlog.Field("bloom1ratio", fmt.Sprintf("%.4f", float64(f.bloom.Ones())/float64(len(f.bloom.Bytes())*8))), slog.Any("bloom1ratio", fmt.Sprintf("%.4f", float64(f.bloom.Ones())/float64(len(f.bloom.Bytes())*8))),
} )
xlog.Print("training done", fields...)
return nil return nil
} }
@ -742,7 +741,7 @@ func (f *Filter) TrainDirs(hamDir, sentDir, spamDir string, hamFiles, sentFiles,
func (f *Filter) fileSize(p string) int { func (f *Filter) fileSize(p string) int {
fi, err := os.Stat(p) fi, err := os.Stat(p)
if err != nil { if err != nil {
f.log.Infox("stat", err, mlog.Field("path", p)) f.log.Infox("stat", err, slog.Any("path", p))
return 0 return 0
} }
return int(fi.Size()) return int(fi.Size())

2
junk/filter_test.go
View file

@ -32,7 +32,7 @@ func tlistdir(t *testing.T, name string) []string {
} }
func TestFilter(t *testing.T) { func TestFilter(t *testing.T) {
log := mlog.New("junk") log := mlog.New("junk", nil)
params := Params{ params := Params{
Onegrams: true, Onegrams: true,
Twograms: true, Twograms: true,

2
junk/parse.go
View file

@ -31,7 +31,7 @@ func (f *Filter) tokenizeMail(path string) (bool, map[string]struct{}, error) {
if err != nil { if err != nil {
return false, nil, err return false, nil, err
} }
p, _ := message.EnsurePart(f.log, false, mf, fi.Size()) p, _ := message.EnsurePart(f.log.Logger, false, mf, fi.Size())
words, err := f.ParseMessage(p) words, err := f.ParseMessage(p)
return true, words, err return true, words, err
} }

5
junk/parse_test.go
View file

@ -4,6 +4,8 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/mjl-/mox/mlog"
) )
func FuzzParseMessage(f *testing.F) { func FuzzParseMessage(f *testing.F) {
@ -24,7 +26,8 @@ func FuzzParseMessage(f *testing.F) {
os.Remove(dbPath) os.Remove(dbPath)
os.Remove(bloomPath) os.Remove(bloomPath)
params := Params{Twograms: true} params := Params{Twograms: true}
jf, err := NewFilter(ctxbg, xlog, params, dbPath, bloomPath) log := mlog.New("junk", nil)
jf, err := NewFilter(ctxbg, log, params, dbPath, bloomPath)
if err != nil { if err != nil {
f.Fatalf("new filter: %v", err) f.Fatalf("new filter: %v", err)
} }

30
localserve.go
View file

@ -20,6 +20,7 @@ import (
"time" "time"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slog"
"github.com/mjl-/sconf" "github.com/mjl-/sconf"
@ -75,18 +76,17 @@ during those commands instead of during "data".
c.Usage() c.Usage()
} }
log := mlog.New("localserve") log := c.log
mox.FilesImmediate = true mox.FilesImmediate = true
if initOnly { if initOnly {
if _, err := os.Stat(dir); err == nil { if _, err := os.Stat(dir); err == nil {
log.Print("warning: directory for configuration files already exists, continuing") log.Print("warning: directory for configuration files already exists, continuing")
} }
log.Print("creating mox localserve config", mlog.Field("dir", dir)) log.Print("creating mox localserve config", slog.String("dir", dir))
err := writeLocalConfig(log, dir, ip) err := writeLocalConfig(log, dir, ip)
if err != nil { if err != nil {
log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir)) log.Fatalx("creating mox localserve config", err, slog.String("dir", dir))
} }
return return
} }
@ -96,12 +96,12 @@ during those commands instead of during "data".
if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
err := writeLocalConfig(log, dir, ip) err := writeLocalConfig(log, dir, ip)
if err != nil { if err != nil {
log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir)) log.Fatalx("creating mox localserve config", err, slog.String("dir", dir))
} }
} else if err != nil { } else if err != nil {
log.Fatalx("stat config dir", err, mlog.Field("dir", dir)) log.Fatalx("stat config dir", err, slog.String("dir", dir))
} else if err := localLoadConfig(log, dir); err != nil { } else if err := localLoadConfig(log, dir); err != nil {
log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, mlog.Field("dir", dir)) log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, slog.String("dir", dir))
} else if ip != "" { } else if ip != "" {
log.Fatal("can only use -ip when writing a new config file") log.Fatal("can only use -ip when writing a new config file")
} else { } else {
@ -112,7 +112,7 @@ during those commands instead of during "data".
mox.Conf.Log[""] = level mox.Conf.Log[""] = level
mlog.SetConfig(mox.Conf.Log) mlog.SetConfig(mox.Conf.Log)
} else if loglevel != "" && !ok { } else if loglevel != "" && !ok {
log.Fatal("unknown loglevel", mlog.Field("loglevel", loglevel)) log.Fatal("unknown loglevel", slog.String("loglevel", loglevel))
} }
// Initialize receivedid. // Initialize receivedid.
@ -201,7 +201,7 @@ during those commands instead of during "data".
sigc := make(chan os.Signal, 1) sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
sig := <-sigc sig := <-sigc
log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig)) log.Print("shutting down, waiting max 3s for existing connections", slog.Any("signal", sig))
shutdown(log) shutdown(log)
if num, ok := sig.(syscall.Signal); ok { if num, ok := sig.(syscall.Signal); ok {
os.Exit(int(num)) os.Exit(int(num))
@ -210,7 +210,7 @@ during those commands instead of during "data".
} }
} }
func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) { func writeLocalConfig(log mlog.Log, dir, ip string) (rerr error) {
defer func() { defer func() {
x := recover() x := recover()
if x != nil { if x != nil {
@ -220,7 +220,7 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
} }
if rerr != nil { if rerr != nil {
err := os.RemoveAll(dir) err := os.RemoveAll(dir)
log.Check(err, "removing config directory", mlog.Field("dir", dir)) log.Check(err, "removing config directory", slog.String("dir", dir))
} }
}() }()
@ -430,10 +430,10 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
xcheck(err, "loading config") xcheck(err, "loading config")
// Set password on account. // Set password on account.
a, _, err := store.OpenEmail("mox@localhost") a, _, err := store.OpenEmail(log, "mox@localhost")
xcheck(err, "opening account to set password") xcheck(err, "opening account to set password")
password := "moxmoxmox" password := "moxmoxmox"
err = a.SetPassword(password) err = a.SetPassword(log, password)
xcheck(err, "setting password") xcheck(err, "setting password")
err = a.Close() err = a.Close()
xcheck(err, "closing account") xcheck(err, "closing account")
@ -442,10 +442,10 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
return nil return nil
} }
func localLoadConfig(log *mlog.Log, dir string) error { func localLoadConfig(log mlog.Log, dir string) error {
mox.ConfigStaticPath = filepath.Join(dir, "mox.conf") mox.ConfigStaticPath = filepath.Join(dir, "mox.conf")
mox.ConfigDynamicPath = filepath.Join(dir, "domains.conf") mox.ConfigDynamicPath = filepath.Join(dir, "domains.conf")
errs := mox.LoadConfig(context.Background(), true, false) errs := mox.LoadConfig(context.Background(), log, true, false)
if len(errs) > 1 { if len(errs) > 1 {
log.Error("loading config generated config file: multiple errors") log.Error("loading config generated config file: multiple errors")
for _, err := range errs { for _, err := range errs {

111
main.go
View file

@ -32,6 +32,7 @@ import (
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"github.com/mjl-/adns" "github.com/mjl-/adns"
@ -211,6 +212,8 @@ type cmd struct {
params string // Arguments to command. Multiple lines possible. params string // Arguments to command. Multiple lines possible.
help string // Additional explanation. First line is synopsis, the rest is only printed for an explicit help/usage for that command. help string // Additional explanation. First line is synopsis, the rest is only printed for an explicit help/usage for that command.
args []string args []string
log mlog.Log
} }
func (c *cmd) Parse() []string { func (c *cmd) Parse() []string {
@ -388,7 +391,7 @@ func mustLoadConfig() {
mox.Conf.Log[""] = level mox.Conf.Log[""] = level
mlog.SetConfig(mox.Conf.Log) mlog.SetConfig(mox.Conf.Log)
} else if loglevel != "" && !ok { } else if loglevel != "" && !ok {
log.Fatal("unknown loglevel", mlog.Field("loglevel", loglevel)) log.Fatal("unknown loglevel", slog.String("loglevel", loglevel))
} }
if pedantic { if pedantic {
moxvar.Pedantic = true moxvar.Pedantic = true
@ -413,6 +416,7 @@ func main() {
c := &cmd{ c := &cmd{
flag: flag.NewFlagSet("sendmail", flag.ExitOnError), flag: flag.NewFlagSet("sendmail", flag.ExitOnError),
flagArgs: os.Args[1:], flagArgs: os.Args[1:],
log: mlog.New("sendmail", nil),
} }
cmdSendmail(c) cmdSendmail(c)
return return
@ -464,6 +468,7 @@ next:
} }
c.flag = flag.NewFlagSet("mox "+strings.Join(c.words, " "), flag.ExitOnError) c.flag = flag.NewFlagSet("mox "+strings.Join(c.words, " "), flag.ExitOnError)
c.flagArgs = args[len(c.words):] c.flagArgs = args[len(c.words):]
c.log = mlog.New(strings.Join(c.words, ""), nil)
c.fn(&c) c.fn(&c)
return return
} }
@ -538,7 +543,7 @@ are printed.
mox.FilesImmediate = true mox.FilesImmediate = true
_, errs := mox.ParseConfig(context.Background(), mox.ConfigStaticPath, true, true, false) _, errs := mox.ParseConfig(context.Background(), c.log, mox.ConfigStaticPath, true, true, false)
if len(errs) > 1 { if len(errs) > 1 {
log.Printf("multiple errors:") log.Printf("multiple errors:")
for _, err := range errs { for _, err := range errs {
@ -1596,7 +1601,7 @@ connection.
} }
resolver := dns.StrictResolver{Pkg: "danedial"} resolver := dns.StrictResolver{Pkg: "danedial"}
conn, record, err := dane.Dial(context.Background(), resolver, "tcp", args[0], allowedUsages) conn, record, err := dane.Dial(context.Background(), c.log.Logger, resolver, "tcp", args[0], allowedUsages)
xcheckf(err, "dial") xcheckf(err, "dial")
log.Printf("(connected, verified with %s)", record) log.Printf("(connected, verified with %s)", record)
@ -1644,8 +1649,6 @@ sharing most of its code.
origNextHop, err := dns.ParseDomain(args[0]) origNextHop, err := dns.ParseDomain(args[0])
xcheckf(err, "parse domain") xcheckf(err, "parse domain")
clog := mlog.New("danedialmx")
ctxbg := context.Background() ctxbg := context.Background()
resolver := dns.StrictResolver{} resolver := dns.StrictResolver{}
@ -1655,7 +1658,7 @@ sharing most of its code.
var hosts []dns.IPDomain var hosts []dns.IPDomain
if len(args) == 1 { if len(args) == 1 {
var permanent bool var permanent bool
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err = smtpclient.GatherDestinations(ctxbg, clog, resolver, dns.IPDomain{Domain: origNextHop}) haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err = smtpclient.GatherDestinations(ctxbg, c.log.Logger, resolver, dns.IPDomain{Domain: origNextHop})
status := "temporary" status := "temporary"
if permanent { if permanent {
status = "permanent" status = "permanent"
@ -1706,7 +1709,7 @@ sharing most of its code.
log.Printf("attempting to connect to %s", host) log.Printf("attempting to connect to %s", host)
authentic, expandedAuthentic, expandedHost, ips, _, err := smtpclient.GatherIPs(ctxbg, clog, resolver, host, dialedIPs) authentic, expandedAuthentic, expandedHost, ips, _, err := smtpclient.GatherIPs(ctxbg, c.log.Logger, resolver, host, dialedIPs)
if err != nil { if err != nil {
log.Printf("resolving ips for %s: %v, skipping", host, err) log.Printf("resolving ips for %s: %v, skipping", host, err)
continue continue
@ -1724,7 +1727,7 @@ sharing most of its code.
} }
log.Printf("host %s resolved to ips %s, looking up tlsa records", host, ips) log.Printf("host %s resolved to ips %s, looking up tlsa records", host, ips)
daneRequired, daneRecords, tlsaBaseDomain, err := smtpclient.GatherTLSA(ctxbg, clog, resolver, host.Domain, expandedAuthentic, expandedHost) daneRequired, daneRecords, tlsaBaseDomain, err := smtpclient.GatherTLSA(ctxbg, c.log.Logger, resolver, host.Domain, expandedAuthentic, expandedHost)
if err != nil { if err != nil {
log.Printf("looking up tlsa records: %s, skipping", err) log.Printf("looking up tlsa records: %s, skipping", err)
continue continue
@ -1753,7 +1756,7 @@ sharing most of its code.
log.Printf("gathered valid tls certificate names for potential verification with dane-ta: %s", strings.Join(l, ", ")) log.Printf("gathered valid tls certificate names for potential verification with dane-ta: %s", strings.Join(l, ", "))
dialer := &net.Dialer{Timeout: 5 * time.Second} dialer := &net.Dialer{Timeout: 5 * time.Second}
conn, _, err := smtpclient.Dial(ctxbg, clog, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs) conn, _, err := smtpclient.Dial(ctxbg, c.log.Logger, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs)
if err != nil { if err != nil {
log.Printf("dial %s: %v, skipping", expandedHost, err) log.Printf("dial %s: %v, skipping", expandedHost, err)
continue continue
@ -1768,7 +1771,7 @@ sharing most of its code.
RootCAs: mox.Conf.Static.TLS.CertPool, RootCAs: mox.Conf.Static.TLS.CertPool,
} }
tlsPKIX := false tlsPKIX := false
sc, err := smtpclient.New(ctxbg, clog, conn, tlsMode, tlsPKIX, ehloDomain, tlsHostnames[0], opts) sc, err := smtpclient.New(ctxbg, c.log.Logger, conn, tlsMode, tlsPKIX, ehloDomain, tlsHostnames[0], opts)
if err != nil { if err != nil {
log.Printf("setting up smtp session: %v, skipping", err) log.Printf("setting up smtp session: %v, skipping", err)
conn.Close() conn.Close()
@ -2175,7 +2178,7 @@ that was passed.
msgf, err := os.Open(args[0]) msgf, err := os.Open(args[0])
xcheckf(err, "open message") xcheckf(err, "open message")
results, err := dkim.Verify(context.Background(), dns.StrictResolver{}, false, dkim.DefaultPolicy, msgf, true) results, err := dkim.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, false, dkim.DefaultPolicy, msgf, true)
xcheckf(err, "dkim verify") xcheckf(err, "dkim verify")
for _, result := range results { for _, result := range results {
@ -2214,13 +2217,11 @@ headers prepended.
c.Usage() c.Usage()
} }
clog := mlog.New("dkimsign")
msgf, err := os.Open(args[0]) msgf, err := os.Open(args[0])
xcheckf(err, "open message") xcheckf(err, "open message")
defer msgf.Close() defer msgf.Close()
p, err := message.Parse(clog, true, msgf) p, err := message.Parse(c.log.Logger, true, msgf)
xcheckf(err, "parsing message") xcheckf(err, "parsing message")
if len(p.Envelope.From) != 1 { if len(p.Envelope.From) != 1 {
@ -2237,7 +2238,7 @@ headers prepended.
log.Fatalf("domain %s not configured", dom) log.Fatalf("domain %s not configured", dom)
} }
headers, err := dkim.Sign(context.Background(), localpart, dom, domConf.DKIM, false, msgf) headers, err := dkim.Sign(context.Background(), c.log.Logger, localpart, dom, domConf.DKIM, false, msgf)
xcheckf(err, "signing message with dkim") xcheckf(err, "signing message with dkim")
if headers == "" { if headers == "" {
log.Fatalf("no DKIM configured for domain %s", dom) log.Fatalf("no DKIM configured for domain %s", dom)
@ -2259,7 +2260,7 @@ func cmdDKIMLookup(c *cmd) {
selector := xparseDomain(args[0], "selector") selector := xparseDomain(args[0], "selector")
domain := xparseDomain(args[1], "domain") domain := xparseDomain(args[1], "domain")
status, record, txt, authentic, err := dkim.Lookup(context.Background(), dns.StrictResolver{}, selector, domain) status, record, txt, authentic, err := dkim.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, selector, domain)
if err != nil { if err != nil {
fmt.Printf("error: %s\n", err) fmt.Printf("error: %s\n", err)
} }
@ -2299,7 +2300,7 @@ func cmdDMARCLookup(c *cmd) {
} }
fromdomain := xparseDomain(args[0], "domain") fromdomain := xparseDomain(args[0], "domain")
_, domain, _, txt, authentic, err := dmarc.Lookup(context.Background(), dns.StrictResolver{}, fromdomain) _, domain, _, txt, authentic, err := dmarc.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, fromdomain)
xcheckf(err, "dmarc lookup domain %s", fromdomain) xcheckf(err, "dmarc lookup domain %s", fromdomain)
fmt.Printf("dmarc record at domain %s: %s\n", domain, txt) fmt.Printf("dmarc record at domain %s: %s\n", domain, txt)
fmt.Printf("(%s)\n", dnssecStatus(authentic)) fmt.Printf("(%s)\n", dnssecStatus(authentic))
@ -2359,7 +2360,7 @@ can be found in message headers.
if heloDomain != nil { if heloDomain != nil {
spfArgs.HelloDomain = dns.IPDomain{Domain: *heloDomain} spfArgs.HelloDomain = dns.IPDomain{Domain: *heloDomain}
} }
rspf, spfDomain, expl, authentic, err := spf.Verify(context.Background(), dns.StrictResolver{}, spfArgs) rspf, spfDomain, expl, authentic, err := spf.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, spfArgs)
if err != nil { if err != nil {
log.Printf("spf verify: %v (explanation: %q, authentic %v)", err, expl, authentic) log.Printf("spf verify: %v (explanation: %q, authentic %v)", err, expl, authentic)
} else { } else {
@ -2377,17 +2378,17 @@ can be found in message headers.
data, err := io.ReadAll(os.Stdin) data, err := io.ReadAll(os.Stdin)
xcheckf(err, "read message") xcheckf(err, "read message")
dmarcFrom, _, _, err := message.From(mlog.New("dmarcverify"), false, bytes.NewReader(data)) dmarcFrom, _, _, err := message.From(c.log.Logger, false, bytes.NewReader(data))
xcheckf(err, "extract dmarc from message") xcheckf(err, "extract dmarc from message")
const ignoreTestMode = false const ignoreTestMode = false
dkimResults, err := dkim.Verify(context.Background(), dns.StrictResolver{}, true, func(*dkim.Sig) error { return nil }, bytes.NewReader(data), ignoreTestMode) dkimResults, err := dkim.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, true, func(*dkim.Sig) error { return nil }, bytes.NewReader(data), ignoreTestMode)
xcheckf(err, "dkim verify") xcheckf(err, "dkim verify")
for _, r := range dkimResults { for _, r := range dkimResults {
fmt.Printf("dkim result: %q (err %v)\n", r.Status, r.Err) fmt.Printf("dkim result: %q (err %v)\n", r.Status, r.Err)
} }
_, result := dmarc.Verify(context.Background(), dns.StrictResolver{}, dmarcFrom.Domain, dkimResults, spfStatus, spfIdentity, false) _, result := dmarc.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, dmarcFrom.Domain, dkimResults, spfStatus, spfIdentity, false)
xcheckf(result.Err, "dmarc verify") xcheckf(result.Err, "dmarc verify")
fmt.Printf("dmarc from: %s\ndmarc status: %q\ndmarc reject: %v\ncmarc record: %s\n", dmarcFrom, result.Status, result.Reject, result.Record) fmt.Printf("dmarc from: %s\ndmarc status: %q\ndmarc reject: %v\ncmarc record: %s\n", dmarcFrom, result.Status, result.Reject, result.Record)
} }
@ -2408,7 +2409,7 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at
} }
dom := xparseDomain(args[0], "domain") dom := xparseDomain(args[0], "domain")
_, domain, record, txt, authentic, err := dmarc.Lookup(context.Background(), dns.StrictResolver{}, dom) _, domain, record, txt, authentic, err := dmarc.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, dom)
xcheckf(err, "dmarc lookup domain %s", dom) xcheckf(err, "dmarc lookup domain %s", dom)
fmt.Printf("dmarc record at domain %s: %q\n", domain, txt) fmt.Printf("dmarc record at domain %s: %q\n", domain, txt)
fmt.Printf("(%s)\n", dnssecStatus(authentic)) fmt.Printf("(%s)\n", dnssecStatus(authentic))
@ -2439,12 +2440,12 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at
return return
} }
if publicsuffix.Lookup(context.Background(), dom) == publicsuffix.Lookup(context.Background(), destdom) { if publicsuffix.Lookup(context.Background(), c.log.Logger, dom) == publicsuffix.Lookup(context.Background(), c.log.Logger, destdom) {
printResult("pass (same organizational domain)") printResult("pass (same organizational domain)")
return return
} }
accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom) accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), c.log.Logger, dns.StrictResolver{}, domain, destdom)
var txtstr string var txtstr string
txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII) txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII)
if len(txts) == 0 { if len(txts) == 0 {
@ -2486,12 +2487,10 @@ understand email deliverability problems.
c.Usage() c.Usage()
} }
clog := mlog.New("dmarcparsereportmsg")
for _, arg := range args { for _, arg := range args {
f, err := os.Open(arg) f, err := os.Open(arg)
xcheckf(err, "open %q", arg) xcheckf(err, "open %q", arg)
feedback, err := dmarcrpt.ParseMessageReport(clog, f) feedback, err := dmarcrpt.ParseMessageReport(c.log.Logger, f)
xcheckf(err, "parse report in %q", arg) xcheckf(err, "parse report in %q", arg)
meta := feedback.ReportMetadata meta := feedback.ReportMetadata
fmt.Printf("Report: period %s-%s, organisation %q, reportID %q, %s\n", time.Unix(meta.DateRange.Begin, 0).UTC().String(), time.Unix(meta.DateRange.End, 0).UTC().String(), meta.OrgName, meta.ReportID, meta.Email) fmt.Printf("Report: period %s-%s, organisation %q, reportID %q, %s\n", time.Unix(meta.DateRange.Begin, 0).UTC().String(), time.Unix(meta.DateRange.End, 0).UTC().String(), meta.OrgName, meta.ReportID, meta.Email)
@ -2540,11 +2539,9 @@ func cmdDMARCDBAddReport(c *cmd) {
mustLoadConfig() mustLoadConfig()
clog := mlog.New("dmarcdbaddreport")
fromdomain := xparseDomain(args[0], "domain") fromdomain := xparseDomain(args[0], "domain")
fmt.Fprintln(os.Stderr, "reading report message from stdin") fmt.Fprintln(os.Stderr, "reading report message from stdin")
report, err := dmarcrpt.ParseMessageReport(clog, os.Stdin) report, err := dmarcrpt.ParseMessageReport(c.log.Logger, os.Stdin)
xcheckf(err, "parse message") xcheckf(err, "parse message")
err = dmarcdb.AddReport(context.Background(), report, fromdomain) err = dmarcdb.AddReport(context.Background(), report, fromdomain)
xcheckf(err, "add dmarc report") xcheckf(err, "add dmarc report")
@ -2565,7 +2562,7 @@ successfully used TLS, and how what kind of errors occurred otherwise.
} }
d := xparseDomain(args[0], "domain") d := xparseDomain(args[0], "domain")
_, txt, err := tlsrpt.Lookup(context.Background(), dns.StrictResolver{}, d) _, txt, err := tlsrpt.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, d)
xcheckf(err, "tlsrpt lookup for %s", d) xcheckf(err, "tlsrpt lookup for %s", d)
fmt.Println(txt) fmt.Println(txt)
} }
@ -2581,12 +2578,10 @@ The report is printed in formatted JSON.
c.Usage() c.Usage()
} }
clog := mlog.New("tlsrptparsereportmsg")
for _, arg := range args { for _, arg := range args {
f, err := os.Open(arg) f, err := os.Open(arg)
xcheckf(err, "open %q", arg) xcheckf(err, "open %q", arg)
report, err := tlsrpt.ParseMessage(clog, f) report, err := tlsrpt.ParseMessage(c.log.Logger, f)
xcheckf(err, "parse report in %q", arg) xcheckf(err, "parse report in %q", arg)
// todo future: only print the highlights? // todo future: only print the highlights?
enc := json.NewEncoder(os.Stdout) enc := json.NewEncoder(os.Stdout)
@ -2622,7 +2617,7 @@ printed.
LocalIP: net.ParseIP("127.0.0.1"), LocalIP: net.ParseIP("127.0.0.1"),
LocalHostname: dns.Domain{ASCII: "localhost"}, LocalHostname: dns.Domain{ASCII: "localhost"},
} }
r, _, explanation, authentic, err := spf.Verify(context.Background(), dns.StrictResolver{}, spfargs) r, _, explanation, authentic, err := spf.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, spfargs)
if err != nil { if err != nil {
fmt.Printf("error: %s\n", err) fmt.Printf("error: %s\n", err)
} }
@ -2656,7 +2651,7 @@ func cmdSPFLookup(c *cmd) {
} }
domain := xparseDomain(args[0], "domain") domain := xparseDomain(args[0], "domain")
_, txt, _, authentic, err := spf.Lookup(context.Background(), dns.StrictResolver{}, domain) _, txt, _, authentic, err := spf.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, domain)
xcheckf(err, "spf lookup for %s", domain) xcheckf(err, "spf lookup for %s", domain)
fmt.Println(txt) fmt.Println(txt)
fmt.Printf("(%s)\n", dnssecStatus(authentic)) fmt.Printf("(%s)\n", dnssecStatus(authentic))
@ -2680,7 +2675,7 @@ should be used, and how long the policy can be cached.
domain := xparseDomain(args[0], "domain") domain := xparseDomain(args[0], "domain")
record, policy, _, err := mtasts.Get(context.Background(), dns.StrictResolver{}, domain) record, policy, _, err := mtasts.Get(context.Background(), c.log.Logger, dns.StrictResolver{}, domain)
if err != nil { if err != nil {
fmt.Printf("error: %s\n", err) fmt.Printf("error: %s\n", err)
} }
@ -2729,13 +2724,11 @@ func cmdTLSRPTDBAddReport(c *cmd) {
mustLoadConfig() mustLoadConfig()
clog := mlog.New("tlsrptdbaddreport")
// First read message, to get the From-header. Then parse it as TLSRPT. // First read message, to get the From-header. Then parse it as TLSRPT.
fmt.Fprintln(os.Stderr, "reading report message from stdin") fmt.Fprintln(os.Stderr, "reading report message from stdin")
buf, err := io.ReadAll(os.Stdin) buf, err := io.ReadAll(os.Stdin)
xcheckf(err, "reading message") xcheckf(err, "reading message")
part, err := message.Parse(clog, true, bytes.NewReader(buf)) part, err := message.Parse(c.log.Logger, true, bytes.NewReader(buf))
xcheckf(err, "parsing message") xcheckf(err, "parsing message")
if part.Envelope == nil || len(part.Envelope.From) != 1 { if part.Envelope == nil || len(part.Envelope.From) != 1 {
log.Fatalf("message must have one From-header") log.Fatalf("message must have one From-header")
@ -2743,11 +2736,11 @@ func cmdTLSRPTDBAddReport(c *cmd) {
from := part.Envelope.From[0] from := part.Envelope.From[0]
domain := xparseDomain(from.Host, "domain") domain := xparseDomain(from.Host, "domain")
report, err := tlsrpt.ParseMessage(clog, bytes.NewReader(buf)) report, err := tlsrpt.ParseMessage(c.log.Logger, bytes.NewReader(buf))
xcheckf(err, "parsing tls report in message") xcheckf(err, "parsing tls report in message")
mailfrom := from.User + "@" + from.Host // todo future: should escape and such mailfrom := from.User + "@" + from.Host // todo future: should escape and such
err = tlsrptdb.AddReport(context.Background(), domain, mailfrom, hostReport, report) err = tlsrptdb.AddReport(context.Background(), c.log, domain, mailfrom, hostReport, report)
xcheckf(err, "add tls report to database") xcheckf(err, "add tls report to database")
} }
@ -2766,7 +2759,7 @@ URL with more information.
zone := xparseDomain(args[0], "zone") zone := xparseDomain(args[0], "zone")
ip := xparseIP(args[1], "ip") ip := xparseIP(args[1], "ip")
status, explanation, err := dnsbl.Lookup(context.Background(), dns.StrictResolver{}, zone, ip) status, explanation, err := dnsbl.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, zone, ip)
fmt.Printf("status: %s\n", status) fmt.Printf("status: %s\n", status)
if status == dnsbl.StatusFail { if status == dnsbl.StatusFail {
fmt.Printf("explanation: %q\n", explanation) fmt.Printf("explanation: %q\n", explanation)
@ -2789,7 +2782,7 @@ The health of a DNS blocklist can be checked by querying for 127.0.0.1 and
} }
zone := xparseDomain(args[0], "zone") zone := xparseDomain(args[0], "zone")
err := dnsbl.CheckHealth(context.Background(), dns.StrictResolver{}, zone) err := dnsbl.CheckHealth(context.Background(), c.log.Logger, dns.StrictResolver{}, zone)
xcheckf(err, "unhealthy") xcheckf(err, "unhealthy")
fmt.Println("healthy") fmt.Println("healthy")
} }
@ -2814,12 +2807,12 @@ printed.
fmt.Printf("last known version: %s\n", lastknown) fmt.Printf("last known version: %s\n", lastknown)
fmt.Printf("current version: %s\n", current) fmt.Printf("current version: %s\n", current)
} }
latest, _, err := updates.Lookup(context.Background(), dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain}) latest, _, err := updates.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain})
xcheckf(err, "lookup of latest version") xcheckf(err, "lookup of latest version")
fmt.Printf("latest version: %s\n", latest) fmt.Printf("latest version: %s\n", latest)
if latest.After(current) { if latest.After(current) {
changelog, err := updates.FetchChangelog(context.Background(), changelogURL, current, changelogPubKey) changelog, err := updates.FetchChangelog(context.Background(), c.log.Logger, changelogURL, current, changelogPubKey)
xcheckf(err, "fetching changelog") xcheckf(err, "fetching changelog")
if len(changelog.Changes) == 0 { if len(changelog.Changes) == 0 {
log.Printf("no changes in changelog") log.Printf("no changes in changelog")
@ -2884,7 +2877,7 @@ open, or is not running.
} }
mustLoadConfig() mustLoadConfig()
a, err := store.OpenAccount(args[0]) a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account") xcheckf(err, "open account")
defer func() { defer func() {
if err := a.Close(); err != nil { if err := a.Close(); err != nil {
@ -2942,7 +2935,7 @@ open, or is not running.
} }
mustLoadConfig() mustLoadConfig()
a, err := store.OpenAccount(args[0]) a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account") xcheckf(err, "open account")
defer func() { defer func() {
if err := a.Close(); err != nil { if err := a.Close(); err != nil {
@ -3036,7 +3029,7 @@ open, or is not running.
} }
mustLoadConfig() mustLoadConfig()
a, err := store.OpenAccount(args[0]) a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account") xcheckf(err, "open account")
defer func() { defer func() {
if err := a.Close(); err != nil { if err := a.Close(); err != nil {
@ -3156,10 +3149,8 @@ func cmdEnsureParsed(c *cmd) {
c.Usage() c.Usage()
} }
clog := mlog.New("ensureparsed")
mustLoadConfig() mustLoadConfig()
a, err := store.OpenAccount(args[0]) a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account") xcheckf(err, "open account")
defer func() { defer func() {
if err := a.Close(); err != nil { if err := a.Close(); err != nil {
@ -3180,7 +3171,7 @@ func cmdEnsureParsed(c *cmd) {
} }
for _, m := range l { for _, m := range l {
mr := a.MessageReader(m) mr := a.MessageReader(m)
p, err := message.EnsurePart(clog, false, mr, m.Size) p, err := message.EnsurePart(c.log.Logger, false, mr, m.Size)
if err != nil { if err != nil {
log.Printf("parsing message %d: %v (continuing)", m.ID, err) log.Printf("parsing message %d: %v (continuing)", m.ID, err)
} }
@ -3233,15 +3224,13 @@ func cmdMessageParse(c *cmd) {
c.Usage() c.Usage()
} }
clog := mlog.New("messageparse")
f, err := os.Open(args[0]) f, err := os.Open(args[0])
xcheckf(err, "open") xcheckf(err, "open")
defer f.Close() defer f.Close()
part, err := message.Parse(clog, false, f) part, err := message.Parse(c.log.Logger, false, f)
xcheckf(err, "parsing message") xcheckf(err, "parsing message")
err = part.Walk(clog, nil) err = part.Walk(c.log.Logger, nil)
xcheckf(err, "parsing nested parts") xcheckf(err, "parsing nested parts")
enc := json.NewEncoder(os.Stdout) enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", "\t") enc.SetIndent("", "\t")
@ -3262,15 +3251,13 @@ Opens database files directly, not going through a running mox instance.
c.Usage() c.Usage()
} }
clog := mlog.New("openaccounts")
dataDir := filepath.Clean(args[0]) dataDir := filepath.Clean(args[0])
for _, accName := range args[1:] { for _, accName := range args[1:] {
accDir := filepath.Join(dataDir, "accounts", accName) accDir := filepath.Join(dataDir, "accounts", accName)
log.Printf("opening account %s...", accDir) log.Printf("opening account %s...", accDir)
a, err := store.OpenAccountDB(accDir, accName) a, err := store.OpenAccountDB(c.log, accDir, accName)
xcheckf(err, "open account %s", accName) xcheckf(err, "open account %s", accName)
err = a.ThreadingWait(clog) err = a.ThreadingWait(c.log)
xcheckf(err, "wait for threading upgrade to complete for %s", accName) xcheckf(err, "wait for threading upgrade to complete for %s", accName)
err = a.Close() err = a.Close()
xcheckf(err, "close account %s", accName) xcheckf(err, "close account %s", accName)
@ -3363,7 +3350,7 @@ Opens database files directly, not going through a running mox instance.
for _, accName := range args[1:] { for _, accName := range args[1:] {
accDir := filepath.Join(dataDir, "accounts", accName) accDir := filepath.Join(dataDir, "accounts", accName)
log.Printf("opening account %s...", accDir) log.Printf("opening account %s...", accDir)
a, err := store.OpenAccountDB(accDir, accName) a, err := store.OpenAccountDB(c.log, accDir, accName)
xcheckf(err, "open account %s", accName) xcheckf(err, "open account %s", accName)
prepareMessages := func(in, out chan moxio.Work[store.Message, threadPrep]) { prepareMessages := func(in, out chan moxio.Work[store.Message, threadPrep]) {

8
message/from.go
View file

@ -5,6 +5,8 @@ import (
"io" "io"
"net/textproto" "net/textproto"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
@ -17,12 +19,14 @@ import (
// From headers may be present. From returns an error if there is not exactly // From headers may be present. From returns an error if there is not exactly
// one address. This address can be used for evaluating a DMARC policy against // one address. This address can be used for evaluating a DMARC policy against
// SPF and DKIM results. // SPF and DKIM results.
func From(log *mlog.Log, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) { func From(elog *slog.Logger, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) {
log := mlog.New("message", elog)
// ../rfc/7489:1243 // ../rfc/7489:1243
// todo: only allow utf8 if enabled in session/message? // todo: only allow utf8 if enabled in session/message?
p, err := Parse(log, strict, r) p, err := Parse(log.Logger, strict, r)
if err != nil { if err != nil {
// todo: should we continue with p, perhaps headers can be parsed? // todo: should we continue with p, perhaps headers can be parsed?
return raddr, nil, nil, fmt.Errorf("parsing message: %v", err) return raddr, nil, nil, fmt.Errorf("parsing message: %v", err)

39
message/part.go
View file

@ -21,6 +21,7 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"golang.org/x/text/encoding/ianaindex" "golang.org/x/text/encoding/ianaindex"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
@ -111,7 +112,8 @@ type Address struct {
// //
// If strict is set, fewer attempts are made to continue parsing when errors are // If strict is set, fewer attempts are made to continue parsing when errors are
// encountered, such as with invalid content-type headers or bare carriage returns. // encountered, such as with invalid content-type headers or bare carriage returns.
func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) { func Parse(elog *slog.Logger, strict bool, r io.ReaderAt) (Part, error) {
log := mlog.New("message", elog)
return newPart(log, strict, r, 0, nil) return newPart(log, strict, r, 0, nil)
} }
@ -122,10 +124,11 @@ func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) {
// //
// If strict is set, fewer attempts are made to continue parsing when errors are // If strict is set, fewer attempts are made to continue parsing when errors are
// encountered, such as with invalid content-type headers or bare carriage returns. // encountered, such as with invalid content-type headers or bare carriage returns.
func EnsurePart(log *mlog.Log, strict bool, r io.ReaderAt, size int64) (Part, error) { func EnsurePart(elog *slog.Logger, strict bool, r io.ReaderAt, size int64) (Part, error) {
p, err := Parse(log, strict, r) log := mlog.New("message", elog)
p, err := Parse(log.Logger, strict, r)
if err == nil { if err == nil {
err = p.Walk(log, nil) err = p.Walk(log.Logger, nil)
} }
if err != nil { if err != nil {
np, err2 := fallbackPart(p, r, size) np, err2 := fallbackPart(p, r, size)
@ -185,7 +188,9 @@ func (p *Part) SetMessageReaderAt() error {
} }
// Walk through message, decoding along the way, and collecting mime part offsets and sizes, and line counts. // Walk through message, decoding along the way, and collecting mime part offsets and sizes, and line counts.
func (p *Part) Walk(log *mlog.Log, parent *Part) error { func (p *Part) Walk(elog *slog.Logger, parent *Part) error {
log := mlog.New("message", elog)
if len(p.bound) == 0 { if len(p.bound) == 0 {
if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") { if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {
// todo: don't read whole submessage in memory... // todo: don't read whole submessage in memory...
@ -194,11 +199,11 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error {
return err return err
} }
br := bytes.NewReader(buf) br := bytes.NewReader(buf)
mp, err := Parse(log, p.strict, br) mp, err := Parse(log.Logger, p.strict, br)
if err != nil { if err != nil {
return fmt.Errorf("parsing embedded message: %w", err) return fmt.Errorf("parsing embedded message: %w", err)
} }
if err := mp.Walk(log, nil); err != nil { if err := mp.Walk(log.Logger, nil); err != nil {
// If this is a DSN and we are not in pedantic mode, accept unexpected end of // If this is a DSN and we are not in pedantic mode, accept unexpected end of
// message. This is quite common because MTA's sometimes just truncate the original // message. This is quite common because MTA's sometimes just truncate the original
// message in a place that makes the message invalid. // message in a place that makes the message invalid.
@ -220,14 +225,14 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error {
} }
for { for {
pp, err := p.ParseNextPart(log) pp, err := p.ParseNextPart(log.Logger)
if err == io.EOF { if err == io.EOF {
return nil return nil
} }
if err != nil { if err != nil {
return err return err
} }
if err := pp.Walk(log, p); err != nil { if err := pp.Walk(log.Logger, p); err != nil {
return err return err
} }
} }
@ -241,7 +246,7 @@ func (p *Part) String() string {
// newPart parses a new part, which can be the top-level message. // newPart parses a new part, which can be the top-level message.
// offset is the bound offset for parts, and the start of message for top-level messages. parent indicates if this is a top-level message or sub-part. // offset is the bound offset for parts, and the start of message for top-level messages. parent indicates if this is a top-level message or sub-part.
// If an error occurs, p's exported values can still be relevant. EnsurePart uses these values. // If an error occurs, p's exported values can still be relevant. EnsurePart uses these values.
func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) { func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) {
if r == nil { if r == nil {
panic("nil reader") panic("nil reader")
} }
@ -325,14 +330,14 @@ func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Pa
p.MediaType = "APPLICATION" p.MediaType = "APPLICATION"
p.MediaSubType = "OCTET-STREAM" p.MediaSubType = "OCTET-STREAM"
} }
log.Debugx("malformed content-type, attempting to recover and continuing", err, mlog.Field("contenttype", p.header.Get("Content-Type")), mlog.Field("mediatype", p.MediaType), mlog.Field("mediasubtype", p.MediaSubType)) log.Debugx("malformed content-type, attempting to recover and continuing", err, slog.String("contenttype", p.header.Get("Content-Type")), slog.String("mediatype", p.MediaType), slog.String("mediasubtype", p.MediaSubType))
} else if mt != "" { } else if mt != "" {
t := strings.SplitN(strings.ToUpper(mt), "/", 2) t := strings.SplitN(strings.ToUpper(mt), "/", 2)
if len(t) != 2 { if len(t) != 2 {
if moxvar.Pedantic || strict { if moxvar.Pedantic || strict {
return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct) return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct)
} }
log.Debug("malformed media-type, ignoring and continuing", mlog.Field("type", mt)) log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt))
p.MediaType = "APPLICATION" p.MediaType = "APPLICATION"
p.MediaSubType = "OCTET-STREAM" p.MediaSubType = "OCTET-STREAM"
} else { } else {
@ -444,7 +449,7 @@ var wordDecoder = mime.WordDecoder{
}, },
} }
func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) { func parseEnvelope(log mlog.Log, h mail.Header) (*Envelope, error) {
date, _ := h.Date() date, _ := h.Date()
// We currently marshal this field to JSON. But JSON cannot represent all // We currently marshal this field to JSON. But JSON cannot represent all
@ -478,7 +483,7 @@ func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) {
return env, nil return env, nil
} }
func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address { func parseAddressList(log mlog.Log, h mail.Header, k string) []Address {
// todo: possibly work around ios mail generating incorrect q-encoded "phrases" with unencoded double quotes? ../rfc/2047:382 // todo: possibly work around ios mail generating incorrect q-encoded "phrases" with unencoded double quotes? ../rfc/2047:382
l, err := h.AddressList(k) l, err := h.AddressList(k)
if err != nil { if err != nil {
@ -490,7 +495,7 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
var user, host string var user, host string
addr, err := smtp.ParseAddress(a.Address) addr, err := smtp.ParseAddress(a.Address)
if err != nil { if err != nil {
log.Infox("parsing address (continuing)", err, mlog.Field("address", a.Address)) log.Infox("parsing address (continuing)", err, slog.Any("address", a.Address))
} else { } else {
user = addr.Localpart.String() user = addr.Localpart.String()
host = addr.Domain.ASCII host = addr.Domain.ASCII
@ -503,7 +508,9 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
// ParseNextPart parses the next (sub)part of this multipart message. // ParseNextPart parses the next (sub)part of this multipart message.
// ParseNextPart returns io.EOF and a nil part when there are no more parts. // ParseNextPart returns io.EOF and a nil part when there are no more parts.
// Only used for initial parsing of message. Once parsed, use p.Parts. // Only used for initial parsing of message. Once parsed, use p.Parts.
func (p *Part) ParseNextPart(log *mlog.Log) (*Part, error) { func (p *Part) ParseNextPart(elog *slog.Logger) (*Part, error) {
log := mlog.New("message", elog)
if len(p.bound) == 0 { if len(p.bound) == 0 {
return nil, errNotMultipart return nil, errNotMultipart
} }

74
message/part_test.go
View file

@ -15,7 +15,7 @@ import (
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/moxvar"
) )
var xlog = mlog.New("message") var pkglog = mlog.New("message", nil)
func tcheck(t *testing.T, err error, msg string) { func tcheck(t *testing.T, err error, msg string) {
t.Helper() t.Helper()
@ -40,7 +40,7 @@ func tfail(t *testing.T, err, expErr error) {
func TestEmptyHeader(t *testing.T) { func TestEmptyHeader(t *testing.T) {
s := "\r\nx" s := "\r\nx"
p, err := EnsurePart(xlog, true, strings.NewReader(s), int64(len(s))) p, err := EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parse empty headers") tcheck(t, err, "parse empty headers")
buf, err := io.ReadAll(p.Reader()) buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -56,7 +56,7 @@ func TestBadContentType(t *testing.T) {
// Pedantic is like strict. // Pedantic is like strict.
moxvar.Pedantic = true moxvar.Pedantic = true
s := "content-type: text/html;;\r\n\r\ntest" s := "content-type: text/html;;\r\n\r\ntest"
p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType) tfail(t, err, ErrBadContentType)
buf, err := io.ReadAll(p.Reader()) buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -67,7 +67,7 @@ func TestBadContentType(t *testing.T) {
// Strict // Strict
s = "content-type: text/html;;\r\n\r\ntest" s = "content-type: text/html;;\r\n\r\ntest"
p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType) tfail(t, err, ErrBadContentType)
buf, err = io.ReadAll(p.Reader()) buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -77,7 +77,7 @@ func TestBadContentType(t *testing.T) {
// Non-strict but unrecoverable content-type. // Non-strict but unrecoverable content-type.
s = "content-type: not a content type;;\r\n\r\ntest" s = "content-type: not a content type;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type") tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader()) buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -87,7 +87,7 @@ func TestBadContentType(t *testing.T) {
// We try to use only the content-type, typically better than application/octet-stream. // We try to use only the content-type, typically better than application/octet-stream.
s = "content-type: text/html;;\r\n\r\ntest" s = "content-type: text/html;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type") tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader()) buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -97,7 +97,7 @@ func TestBadContentType(t *testing.T) {
// Not recovering multipart, we won't have a boundary. // Not recovering multipart, we won't have a boundary.
s = "content-type: multipart/mixed;;\r\n\r\ntest" s = "content-type: multipart/mixed;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type") tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader()) buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -112,20 +112,20 @@ func TestBareCR(t *testing.T) {
// Pedantic is like strict. // Pedantic is like strict.
moxvar.Pedantic = true moxvar.Pedantic = true
p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader()) _, err = io.ReadAll(p.Reader())
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
moxvar.Pedantic = false moxvar.Pedantic = false
// Strict. // Strict.
p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader()) _, err = io.ReadAll(p.Reader())
tcheck(t, err, "read fallback part without error") tcheck(t, err, "read fallback part without error")
// Non-strict allows bare cr. // Non-strict allows bare cr.
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parse") tcheck(t, err, "parse")
buf, err := io.ReadAll(p.Reader()) buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read") tcheck(t, err, "read")
@ -141,7 +141,7 @@ aGkK
func TestBasic(t *testing.T) { func TestBasic(t *testing.T) {
r := strings.NewReader(basicMsg) r := strings.NewReader(basicMsg)
p, err := Parse(xlog, true, r) p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
buf, err := io.ReadAll(p.RawReader()) buf, err := io.ReadAll(p.RawReader())
@ -176,7 +176,7 @@ Hello Joe, do you think we can meet at 3:30 tomorrow?
func TestBasic2(t *testing.T) { func TestBasic2(t *testing.T) {
r := strings.NewReader(basicMsg2) r := strings.NewReader(basicMsg2)
p, err := Parse(xlog, true, r) p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
buf, err := io.ReadAll(p.RawReader()) buf, err := io.ReadAll(p.RawReader())
@ -196,9 +196,9 @@ func TestBasic2(t *testing.T) {
} }
r = strings.NewReader(basicMsg2) r = strings.NewReader(basicMsg2)
p, err = Parse(xlog, true, r) p, err = Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
err = p.Walk(xlog, nil) err = p.Walk(pkglog.Logger, nil)
tcheck(t, err, "walk") tcheck(t, err, "walk")
if p.RawLineCount != 2 { if p.RawLineCount != 2 {
t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount) t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount)
@ -237,25 +237,25 @@ This is the epilogue. It is also to be ignored.
func TestMime(t *testing.T) { func TestMime(t *testing.T) {
// from ../rfc/2046:1148 // from ../rfc/2046:1148
r := strings.NewReader(mimeMsg) r := strings.NewReader(mimeMsg)
p, err := Parse(xlog, true, r) p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
if len(p.bound) == 0 { if len(p.bound) == 0 {
t.Fatalf("got no bound, expected bound for mime message") t.Fatalf("got no bound, expected bound for mime message")
} }
pp, err := p.ParseNextPart(xlog) pp, err := p.ParseNextPart(pkglog.Logger)
tcheck(t, err, "next part") tcheck(t, err, "next part")
buf, err := io.ReadAll(pp.Reader()) buf, err := io.ReadAll(pp.Reader())
tcheck(t, err, "read all") tcheck(t, err, "read all")
tcompare(t, string(buf), "This is implicitly typed plain US-ASCII text.\r\nIt does NOT end with a linebreak.") tcompare(t, string(buf), "This is implicitly typed plain US-ASCII text.\r\nIt does NOT end with a linebreak.")
pp, err = p.ParseNextPart(xlog) pp, err = p.ParseNextPart(pkglog.Logger)
tcheck(t, err, "next part") tcheck(t, err, "next part")
buf, err = io.ReadAll(pp.Reader()) buf, err = io.ReadAll(pp.Reader())
tcheck(t, err, "read all") tcheck(t, err, "read all")
tcompare(t, string(buf), "This is explicitly typed plain US-ASCII text.\r\nIt DOES end with a linebreak.\r\n") tcompare(t, string(buf), "This is explicitly typed plain US-ASCII text.\r\nIt DOES end with a linebreak.\r\n")
_, err = p.ParseNextPart(xlog) _, err = p.ParseNextPart(pkglog.Logger)
tcompare(t, err, io.EOF) tcompare(t, err, io.EOF)
if len(p.Parts) != 2 { if len(p.Parts) != 2 {
@ -274,17 +274,17 @@ func TestLongLine(t *testing.T) {
for i := range line { for i := range line {
line[i] = 'a' line[i] = 'a'
} }
_, err := Parse(xlog, true, bytes.NewReader(line)) _, err := Parse(pkglog.Logger, true, bytes.NewReader(line))
tfail(t, err, errLineTooLong) tfail(t, err, errLineTooLong)
} }
func TestBareCrLf(t *testing.T) { func TestBareCrLf(t *testing.T) {
parse := func(strict bool, s string) error { parse := func(strict bool, s string) error {
p, err := Parse(xlog, strict, strings.NewReader(s)) p, err := Parse(pkglog.Logger, strict, strings.NewReader(s))
if err != nil { if err != nil {
return err return err
} }
return p.Walk(xlog, nil) return p.Walk(pkglog.Logger, nil)
} }
err := parse(false, "subject: test\ntest\r\n") err := parse(false, "subject: test\ntest\r\n")
tfail(t, err, errBareLF) tfail(t, err, errBareLF)
@ -316,25 +316,25 @@ func TestMissingClosingBoundary(t *testing.T) {
test test
`, "\n", "\r\n") `, "\n", "\r\n")
msg, err := Parse(xlog, false, strings.NewReader(message)) msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
err = walkmsg(&msg) err = walkmsg(&msg)
tfail(t, err, errMissingClosingBoundary) tfail(t, err, errMissingClosingBoundary)
msg, _ = Parse(xlog, false, strings.NewReader(message)) msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
err = msg.Walk(xlog, nil) err = msg.Walk(pkglog.Logger, nil)
tfail(t, err, errMissingClosingBoundary) tfail(t, err, errMissingClosingBoundary)
} }
func TestHeaderEOF(t *testing.T) { func TestHeaderEOF(t *testing.T) {
message := "header: test" message := "header: test"
_, err := Parse(xlog, false, strings.NewReader(message)) _, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tfail(t, err, errUnexpectedEOF) tfail(t, err, errUnexpectedEOF)
} }
func TestBodyEOF(t *testing.T) { func TestBodyEOF(t *testing.T) {
message := "header: test\r\n\r\ntest" message := "header: test\r\n\r\ntest"
msg, err := Parse(xlog, true, strings.NewReader(message)) msg, err := Parse(pkglog.Logger, true, strings.NewReader(message))
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
buf, err := io.ReadAll(msg.Reader()) buf, err := io.ReadAll(msg.Reader())
tcheck(t, err, "read body") tcheck(t, err, "read body")
@ -365,7 +365,7 @@ test
`, "\n", "\r\n") `, "\n", "\r\n")
msg, err := Parse(xlog, false, strings.NewReader(message)) msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
enforceSequential = true enforceSequential = true
defer func() { defer func() {
@ -374,8 +374,8 @@ test
err = walkmsg(&msg) err = walkmsg(&msg)
tcheck(t, err, "walkmsg") tcheck(t, err, "walkmsg")
msg, _ = Parse(xlog, false, strings.NewReader(message)) msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
err = msg.Walk(xlog, nil) err = msg.Walk(pkglog.Logger, nil)
tcheck(t, err, "msg.Walk") tcheck(t, err, "msg.Walk")
} }
@ -452,7 +452,7 @@ Content-Transfer-Encoding: Quoted-printable
--unique-boundary-1-- --unique-boundary-1--
`, "\n", "\r\n") `, "\n", "\r\n")
msg, err := Parse(xlog, true, strings.NewReader(nestedMessage)) msg, err := Parse(pkglog.Logger, true, strings.NewReader(nestedMessage))
tcheck(t, err, "new reader") tcheck(t, err, "new reader")
enforceSequential = true enforceSequential = true
defer func() { defer func() {
@ -477,8 +477,8 @@ Content-Transfer-Encoding: Quoted-printable
t.Fatalf("got %q, expected %q", buf, exp) t.Fatalf("got %q, expected %q", buf, exp)
} }
msg, _ = Parse(xlog, false, strings.NewReader(nestedMessage)) msg, _ = Parse(pkglog.Logger, false, strings.NewReader(nestedMessage))
err = msg.Walk(xlog, nil) err = msg.Walk(pkglog.Logger, nil)
tcheck(t, err, "msg.Walk") tcheck(t, err, "msg.Walk")
} }
@ -518,7 +518,7 @@ func walk(path string) error {
return err return err
} }
defer r.Close() defer r.Close()
msg, err := Parse(xlog, false, r) msg, err := Parse(pkglog.Logger, false, r)
if err != nil { if err != nil {
return err return err
} }
@ -538,7 +538,7 @@ func walkmsg(msg *Part) error {
} }
if msg.MediaType == "MESSAGE" && (msg.MediaSubType == "RFC822" || msg.MediaSubType == "GLOBAL") { if msg.MediaType == "MESSAGE" && (msg.MediaSubType == "RFC822" || msg.MediaSubType == "GLOBAL") {
mp, err := Parse(xlog, false, bytes.NewReader(buf)) mp, err := Parse(pkglog.Logger, false, bytes.NewReader(buf))
if err != nil { if err != nil {
return err return err
} }
@ -566,7 +566,7 @@ func walkmsg(msg *Part) error {
} }
for { for {
pp, err := msg.ParseNextPart(xlog) pp, err := msg.ParseNextPart(pkglog.Logger)
if err == io.EOF { if err == io.EOF {
return nil return nil
} }
@ -585,7 +585,7 @@ func TestEmbedded(t *testing.T) {
tcheck(t, err, "open") tcheck(t, err, "open")
fi, err := f.Stat() fi, err := f.Stat()
tcheck(t, err, "stat") tcheck(t, err, "stat")
_, err = EnsurePart(xlog, false, f, fi.Size()) _, err = EnsurePart(pkglog.Logger, false, f, fi.Size())
tcheck(t, err, "parse") tcheck(t, err, "parse")
} }
@ -594,6 +594,6 @@ func TestEmbedded2(t *testing.T) {
tcheck(t, err, "readfile") tcheck(t, err, "readfile")
buf = bytes.ReplaceAll(buf, []byte("\n"), []byte("\r\n")) buf = bytes.ReplaceAll(buf, []byte("\n"), []byte("\r\n"))
_, err = EnsurePart(xlog, false, bytes.NewReader(buf), int64(len(buf))) _, err = EnsurePart(pkglog.Logger, false, bytes.NewReader(buf), int64(len(buf)))
tfail(t, err, nil) tfail(t, err, nil)
} }

2
message/referencedids_test.go
View file

@ -9,7 +9,7 @@ func TestReferencedIDs(t *testing.T) {
check := func(msg string, expRefs []string) { check := func(msg string, expRefs []string) {
t.Helper() t.Helper()
p, err := Parse(xlog, true, strings.NewReader(msg)) p, err := Parse(pkglog.Logger, true, strings.NewReader(msg))
tcheck(t, err, "parsing message") tcheck(t, err, "parsing message")
h, err := p.Header() h, err := p.Header()

10
metrics/http.go
View file

@ -8,14 +8,14 @@ import (
"os" "os"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
var xlog = mlog.New("metrics")
var ( var (
metricHTTPClient = promauto.NewHistogramVec( metricHTTPClient = promauto.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
@ -34,8 +34,8 @@ var (
// HTTPClientObserve tracks the result of an HTTP transaction in a metric, and // HTTPClientObserve tracks the result of an HTTP transaction in a metric, and
// logs the result. // logs the result.
func HTTPClientObserve(ctx context.Context, pkg, method string, statusCode int, err error, start time.Time) { func HTTPClientObserve(ctx context.Context, log mlog.Log, pkg, method string, statusCode int, err error, start time.Time) {
log := xlog.WithContext(ctx) log = log.WithPkg("metrics")
var result string var result string
switch { switch {
case err == nil: case err == nil:
@ -57,5 +57,5 @@ func HTTPClientObserve(ctx context.Context, pkg, method string, statusCode int,
result = "error" result = "error"
} }
metricHTTPClient.WithLabelValues(pkg, method, result, fmt.Sprintf("%d", statusCode)).Observe(float64(time.Since(start)) / float64(time.Second)) metricHTTPClient.WithLabelValues(pkg, method, result, fmt.Sprintf("%d", statusCode)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("httpclient result", err, mlog.Field("pkg", pkg), mlog.Field("method", method), mlog.Field("code", statusCode), mlog.Field("duration", time.Since(start))) log.Debugx("httpclient result", err, slog.String("pkg", pkg), slog.String("method", method), slog.Int("code", statusCode), slog.Duration("duration", time.Since(start)))
} }

633
mlog/log.go
View file

@ -1,30 +1,21 @@
// Package mlog provides logging with log levels and fields. // Package mlog providers helpers on top of slog.Logger.
// //
// Each log level has a function to log with and without error. // Packages of mox that are fit or use by external code take an *slog.Logger as
// Each such function takes a varargs list of fields (key value pairs) to log. // parameter for logging. Internally, and packages not intended for reuse,
// Variable data should be in fields. Logging strings themselves should be // logging is done with mlog.Log. It providers convenience functions for:
// constant, for easier log processing (e.g. building metrics based on log // logging error values, tracing (protocol messages), uncoditional printing
// messages). // optionally exiting.
// //
// The log levels can be configured per originating package, e.g. smtpclient, // An mlog provides a handler for an mlog.Log for formatting log lines. Lines are
// imapserver. The configuration is application-global, so each Log instance // logged as "logfmt" lines for "mox serve". For command-line tools, the lines are
// uses the same log levels. // printed with colon-separated level, message and error, followed by
// // semicolon-separated attributes.
// Print* should be used for lines that always should be printed, regardless of
// configured log levels. Useful for startup logging and subcommands.
//
// Fatal* stops the program. Its log text is always printed.
package mlog package mlog
// todo: log with source=path:linenumber? and/or stacktrace (perhaps optional)
// todo: should we turn errors logged with an context.Canceled from a level error into level info?
// todo: rethink format. perhaps simply using %#v is more useful for many types?
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/base64" "encoding/base64"
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
@ -32,91 +23,101 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync/atomic" "sync/atomic"
"time"
"golang.org/x/exp/slog"
) )
var noctx = context.Background()
// Logfmt enabled output in logfmt, instead of output more suitable for
// command-line tools. Must be set early in a program lifecycle.
var Logfmt bool var Logfmt bool
type Level int
func (l Level) String() string {
return LevelStrings[l]
}
var LevelStrings = map[Level]string{
LevelPrint: "print",
LevelFatal: "fatal",
LevelError: "error",
LevelInfo: "info",
LevelDebug: "debug",
LevelTrace: "trace",
LevelTraceauth: "traceauth",
LevelTracedata: "tracedata",
}
var Levels = map[string]Level{
"print": LevelPrint,
"fatal": LevelFatal,
"error": LevelError,
"info": LevelInfo,
"debug": LevelDebug,
"trace": LevelTrace,
"traceauth": LevelTraceauth,
"tracedata": LevelTracedata,
}
const (
LevelPrint Level = 0 // Printed regardless of configured log level.
LevelFatal Level = 1 // Printed regardless of configured log level.
LevelError Level = 2
LevelInfo Level = 3
LevelDebug Level = 4
LevelTrace Level = 5
LevelTraceauth Level = 6
LevelTracedata Level = 7
)
// LogStringer is used when formatting field values during logging. If a value // LogStringer is used when formatting field values during logging. If a value
// implements it, LogString is called for the value to log. // implements it, LogString is called for the value to log.
type LogStringer interface { type LogStringer interface {
LogString() string LogString() string
} }
// Holds a map[string]Level, mapping a package (field pkg in logs) to a log level. var lowestLevel atomic.Int32 // For quick initial check.
// The empty string is the default/fallback log level. var config atomic.Pointer[map[string]slog.Level] // For secondary complete check for match.
var config atomic.Value
func init() { func init() {
config.Store(map[string]Level{"": LevelError}) SetConfig(map[string]slog.Level{"": LevelInfo})
} }
// SetConfig atomically sets the new log levels used by all Log instances. // SetConfig atomically sets the new log levels used by all Log instances.
func SetConfig(c map[string]Level) { func SetConfig(c map[string]slog.Level) {
config.Store(c) lowest := c[""]
} for _, l := range c {
if l < lowest {
// Pair is a field/value pair, for use in logged lines. lowest = l
type Pair struct { }
Key string
Value any
}
// Field is a shorthand for making a Pair.
func Field(k string, v any) Pair {
return Pair{k, v}
}
// Log is an instance potentially with its own field/value pair added to any
// logging output.
type Log struct {
fields []Pair
moreFields func() []Pair
}
// New returns a new Log instance. Each log invocation adds field "pkg".
func New(pkg string) *Log {
return &Log{
fields: []Pair{{"pkg", pkg}},
} }
lowestLevel.Store(int32(lowest))
config.Store(&c)
}
var (
// When the configured log level is any of the Trace levels, all protocol messages
// are printed. But protocol "data" (like an email message in the SMTP DATA
// command) is replaced with "..." unless the configured level is LevelTracedata.
// Likewise, protocol messages with authentication data (e.g. plaintext base64
// passwords) are replaced with "***" unless the configured level is LevelTraceauth
// or LevelTracedata.
LevelTracedata = slog.LevelDebug - 8
LevelTraceauth = slog.LevelDebug - 6
LevelTrace = slog.LevelDebug - 4
LevelDebug = slog.LevelDebug
LevelInfo = slog.LevelInfo
LevelError = slog.LevelError
LevelFatal = slog.LevelError + 4 // Printed regardless of configured log level.
LevelPrint = slog.LevelError + 8 // Printed regardless of configured log level.
)
// Levelstrings map log levels to human-readable names.
var LevelStrings = map[slog.Level]string{
LevelTracedata: "tracedata",
LevelTraceauth: "traceauth",
LevelTrace: "trace",
LevelDebug: "debug",
LevelInfo: "info",
LevelError: "error",
LevelFatal: "fatal",
LevelPrint: "print",
}
// Levels map the human-readable log level to a level.
var Levels = map[string]slog.Level{
"tracedata": LevelTracedata,
"traceauth": LevelTraceauth,
"trace": LevelTrace,
"debug": LevelDebug,
"info": LevelInfo,
"error": LevelError,
"fatal": LevelFatal,
"print": LevelPrint,
}
// Log wraps an slog.Logger, providing convenience functions.
type Log struct {
*slog.Logger
}
// New returns a Log that adds a "pkg" attribute. If logger is nil, a new
// Logger is created with a custom handler.
func New(pkg string, logger *slog.Logger) Log {
if logger == nil {
logger = slog.New(&handler{})
}
return Log{logger}.WithPkg(pkg)
}
// WithCid adds a attribute "cid".
// Also see WithContext.
func (l Log) WithCid(cid int64) Log {
return l.With(slog.Int64("cid", cid))
} }
type key string type key string
@ -124,19 +125,13 @@ type key string
// CidKey can be used with context.WithValue to store a "cid" in a context, for logging. // CidKey can be used with context.WithValue to store a "cid" in a context, for logging.
var CidKey key = "cid" var CidKey key = "cid"
// WithCid adds a field "cid".
// Also see WithContext.
func (l *Log) WithCid(cid int64) *Log {
return l.Fields(Pair{"cid", cid})
}
// WithContext adds cid from context, if present. Context are often passed to // WithContext adds cid from context, if present. Context are often passed to
// functions, especially between packages, to pass a "cid" for an operation. At the // functions, especially between packages, to pass a "cid" for an operation. At the
// start of a function (especially if exported) a variable "log" is often // start of a function (especially if exported) a variable "log" is often
// instantiated from a package-level variable "xlog", with WithContext for its cid. // instantiated from a package-level logger, with WithContext for its cid.
// A *Log could be passed instead, but contexts are more pervasive. For the same // Ideally, a Log could be passed instead, but contexts are more pervasive. For the same
// reason WithContext is more common than WithCid. // reason WithContext is more common than WithCid.
func (l *Log) WithContext(ctx context.Context) *Log { func (l Log) WithContext(ctx context.Context) Log {
cidv := ctx.Value(CidKey) cidv := ctx.Value(CidKey)
if cidv == nil { if cidv == nil {
return l return l
@ -145,86 +140,200 @@ func (l *Log) WithContext(ctx context.Context) *Log {
return l.WithCid(cid) return l.WithCid(cid)
} }
// Field adds fields to the logger. Each logged line adds these fields. // With adds attributes to to each logged line.
func (l *Log) Fields(fields ...Pair) *Log { func (l Log) With(attrs ...slog.Attr) Log {
nl := *l return Log{slog.New(l.Logger.Handler().WithAttrs(attrs))}
nl.fields = append(fields, nl.fields...)
return &nl
} }
// MoreFields sets a function on the logger that is called just before logging, // WithPkg ensures pkg is added as attribute to logged lines. If the handler is
// to retrieve additional fields to log. // an mlog handler, pkg is only added if not already the last added package.
func (l *Log) MoreFields(fn func() []Pair) *Log { func (l Log) WithPkg(pkg string) Log {
nl := *l h := l.Logger.Handler()
nl.moreFields = fn if ph, ok := h.(*handler); ok {
return &nl if len(ph.Pkgs) > 0 && ph.Pkgs[len(ph.Pkgs)-1] == pkg {
return l
}
return Log{slog.New(ph.WithPkg(pkg))}
}
return Log{slog.New(h.WithAttrs([]slog.Attr{slog.String("pkg", pkg)}))}
}
// WithFunc sets fn to be called for additional attributes. Fn is only called
// when the line is logged.
// If the underlying handler is not an mlog.handler, this method has no effect.
// Caller must take care of preventing data races.
func (l Log) WithFunc(fn func() []slog.Attr) Log {
h := l.Logger.Handler()
if ph, ok := h.(*handler); ok {
return Log{slog.New(ph.WithFunc(fn))}
}
// Ignored for other handlers, only used internally (smtpserver, imapserver).
return l
} }
// Check logs an error if err is not nil. Intended for logging errors that are good // Check logs an error if err is not nil. Intended for logging errors that are good
// to know, but would not influence program flow. // to know, but would not influence program flow.
func (l *Log) Check(err error, text string, fields ...Pair) { func (l Log) Check(err error, msg string, attrs ...slog.Attr) {
if err != nil { if err != nil {
l.Errorx(text, err, fields...) l.Errorx(msg, err, attrs...)
} }
} }
func (l *Log) Trace(traceLevel Level, text string) bool { func errAttr(err error) slog.Attr {
return l.logx(traceLevel, nil, text) return slog.Any("err", err)
} }
func (l *Log) Fatal(text string, fields ...Pair) { l.Fatalx(text, nil, fields...) } // todo: consider taking a context parameter. it would require all code be refactored. we may want to do this if callers really depend on passing attrs through context. the mox code base does not do that. it makes all call sites more tedious, and requires passing around ctx everywhere, so consider carefully.
func (l *Log) Fatalx(text string, err error, fields ...Pair) {
l.plog(LevelFatal, err, text, fields...) func (l Log) Debug(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelDebug, msg, attrs...)
}
func (l Log) Debugx(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelDebug, msg, attrs...)
}
func (l Log) Info(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelInfo, msg, attrs...)
}
func (l Log) Infox(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelInfo, msg, attrs...)
}
func (l Log) Error(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelError, msg, attrs...)
}
func (l Log) Errorx(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelError, msg, attrs...)
}
func (l Log) Fatal(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelFatal, msg, attrs...)
os.Exit(1) os.Exit(1)
} }
func (l *Log) Print(text string, fields ...Pair) bool { func (l Log) Fatalx(msg string, err error, attrs ...slog.Attr) {
return l.logx(LevelPrint, nil, text, fields...) if err != nil {
} attrs = append([]slog.Attr{errAttr(err)}, attrs...)
func (l *Log) Printx(text string, err error, fields ...Pair) bool { }
return l.logx(LevelPrint, err, text, fields...) l.Logger.LogAttrs(noctx, LevelFatal, msg, attrs...)
os.Exit(1)
} }
func (l *Log) Debug(text string, fields ...Pair) bool { func (l Log) Print(msg string, attrs ...slog.Attr) {
return l.logx(LevelDebug, nil, text, fields...) l.Logger.LogAttrs(noctx, LevelPrint, msg, attrs...)
}
func (l *Log) Debugx(text string, err error, fields ...Pair) bool {
return l.logx(LevelDebug, err, text, fields...)
} }
func (l *Log) Info(text string, fields ...Pair) bool { return l.logx(LevelInfo, nil, text, fields...) } func (l Log) Printx(msg string, err error, attrs ...slog.Attr) {
func (l *Log) Infox(text string, err error, fields ...Pair) bool { if err != nil {
return l.logx(LevelInfo, err, text, fields...) attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelPrint, msg, attrs...)
} }
func (l *Log) Error(text string, fields ...Pair) bool { // Trace logs at trace/traceauth/tracedata level.
return l.logx(LevelError, nil, text, fields...) // If the active log level is any of the trace levels, the data is logged.
} // If level is for tracedata, but the active level doesn't trace data, data is replaced with "...".
func (l *Log) Errorx(text string, err error, fields ...Pair) bool { // If level is for traceauth, but the active level doesn't trace auth, data is replaced with "***".
return l.logx(LevelError, err, text, fields...) func (l Log) Trace(level slog.Level, prefix string, data []byte) {
} h := l.Handler()
if !h.Enabled(noctx, level) {
return
}
ph, ok := h.(*handler)
if !ok {
msg := prefix + string(data)
r := slog.NewRecord(time.Now(), level, msg, 0)
h.Handle(noctx, r)
return
}
filterLevel, ok := ph.configMatch(level)
if !ok {
return
}
func (l *Log) logx(level Level, err error, text string, fields ...Pair) bool { var msg string
if ok, high := l.match(level); ok { if hideData, hideAuth := traceLevel(filterLevel, level); hideData {
// Nothing. msg = prefix + "..."
} else if high >= LevelTrace && level == LevelTraceauth { } else if hideAuth {
text = "***" msg = prefix + "***"
} else if high >= LevelTrace && level == LevelTracedata {
text = "..."
} else { } else {
return false msg = prefix + string(data)
} }
if level > LevelTrace { r := slog.NewRecord(time.Time{}, level, msg, 0)
level = LevelTrace ph.write(filterLevel, r)
}
func traceLevel(level, recordLevel slog.Level) (hideData, hideAuth bool) {
hideData = recordLevel == LevelTracedata && level > LevelTracedata
hideAuth = recordLevel == LevelTraceauth && level > LevelTraceauth
return
}
type handler struct {
Pkgs []string
Attrs []slog.Attr
Group string // Empty or with dot-separated names, ending with a dot.
Fn func() []slog.Attr // Only called when record is actually being logged.
}
func match(minLevel, level slog.Level) bool {
return level >= LevelFatal || level >= minLevel || minLevel <= LevelTrace && level <= LevelTrace
}
func (h *handler) Enabled(ctx context.Context, level slog.Level) bool {
return match(slog.Level(lowestLevel.Load()), level)
}
func (h *handler) configMatch(level slog.Level) (slog.Level, bool) {
c := *config.Load()
for i := len(h.Pkgs) - 1; i >= 0; i-- {
if l, ok := c[h.Pkgs[i]]; ok {
return l, match(l, level)
}
}
l := c[""]
return l, match(l, level)
}
func (h *handler) Handle(ctx context.Context, r slog.Record) error {
l, ok := h.configMatch(r.Level)
if !ok {
return nil
}
if hideData, hideAuth := traceLevel(l, r.Level); hideData {
r.Message = "..."
} else if hideAuth {
r.Message = "***"
}
return h.write(l, r)
}
// Reuse buffers to format log lines into.
var logBuffersStore [32][256]byte
var logBuffers = make(chan []byte, 200)
func init() {
for i := range logBuffersStore {
logBuffers <- logBuffersStore[i][:]
} }
l.plog(level, err, text, fields...)
return true
} }
// escape logfmt string if required, otherwise return original string. // escape logfmt string if required, otherwise return original string.
func logfmtValue(s string) string { func formatString(s string) string {
for _, c := range s { for _, c := range s {
if c == '"' || c == '\\' || c <= ' ' || c == '=' || c >= 0x7f { if c <= ' ' || c == '"' || c == '\\' || c == '=' || c >= 0x7f {
return fmt.Sprintf("%q", s) return fmt.Sprintf("%q", s)
} }
} }
@ -261,6 +370,8 @@ func stringValue(iscid, nested bool, v any) string {
return "" return ""
} }
return "[" + strings.Join(r, ",") + "]" return "[" + strings.Join(r, ",") + "]"
case error:
return r.Error()
} }
rv := reflect.ValueOf(v) rv := reflect.ValueOf(v)
@ -320,100 +431,188 @@ func stringValue(iscid, nested bool, v any) string {
} }
first = false first = false
k := strings.ToLower(t.Field(i).Name) k := strings.ToLower(t.Field(i).Name)
b.WriteString(k + "=" + logfmtValue(vs)) b.WriteString(k + "=" + vs)
} }
return b.String() return b.String()
} }
func (l *Log) plog(level Level, err error, text string, fields ...Pair) { func writeAttr(w io.Writer, separator, group string, a slog.Attr) {
fields = append(l.fields, fields...) switch a.Value.Kind() {
if l.moreFields != nil { case slog.KindGroup:
fields = append(fields, l.moreFields()...) if group != "" {
group += "."
}
group += a.Key
for _, a := range a.Value.Group() {
writeAttr(w, separator, group, a)
}
return
default:
var vv any
if a.Value.Kind() == slog.KindLogValuer {
vv = a.Value.Resolve().Any()
} else {
vv = a.Value.Any()
}
s := stringValue(a.Key == "cid", false, vv)
fmt.Fprint(w, separator, group, a.Key, "=", formatString(s))
} }
// We build up a buffer so we can do a single atomic write of the data. Otherwise partial log lines may interleaf.
b := &bytes.Buffer{}
if Logfmt {
fmt.Fprintf(b, "l=%s m=%s", LevelStrings[level], logfmtValue(text))
if err != nil {
fmt.Fprintf(b, " err=%s", logfmtValue(err.Error()))
}
for i := 0; i < len(fields); i++ {
kv := fields[i]
fmt.Fprintf(b, " %s=%s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
}
b.WriteString("\n")
} else {
fmt.Fprintf(b, "%s: %s", LevelStrings[level], logfmtValue(text))
if err != nil {
fmt.Fprintf(b, ": %s", logfmtValue(err.Error()))
}
if len(fields) > 0 {
fmt.Fprint(b, " (")
for i := 0; i < len(fields); i++ {
if i > 0 {
fmt.Fprint(b, "; ")
}
kv := fields[i]
fmt.Fprintf(b, "%s: %s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
}
fmt.Fprint(b, ")")
}
b.WriteString("\n")
}
os.Stderr.Write(b.Bytes())
} }
func (l *Log) match(level Level) (bool, Level) { func (h *handler) write(l slog.Level, r slog.Record) error {
if level == LevelPrint || level == LevelFatal { // Reuse a buffer, or temporarily allocate a new one.
return true, level var buf []byte
select {
case buf = <-logBuffers:
defer func() {
logBuffers <- buf
}()
default:
buf = make([]byte, 128)
} }
cl := config.Load().(map[string]Level) b := bytes.NewBuffer(buf[:0])
eb := &errWriter{b, nil}
seen := false if Logfmt {
var high Level var wrotePkgs bool
for _, kv := range l.fields { ensurePkgs := func() {
if kv.Key != "pkg" { if !wrotePkgs {
continue wrotePkgs = true
for _, pkg := range h.Pkgs {
writeAttr(eb, " ", "", slog.String("pkg", pkg))
}
}
} }
pkg, ok := kv.Value.(string)
if !ok { fmt.Fprint(eb, "l=", LevelStrings[r.Level], " m=")
continue fmt.Fprintf(eb, "%q", r.Message)
n := 0
r.Attrs(func(a slog.Attr) bool {
if n > 0 || a.Key != "err" || h.Group != "" {
ensurePkgs()
}
writeAttr(eb, " ", h.Group, a)
n++
return true
})
ensurePkgs()
for _, a := range h.Attrs {
writeAttr(eb, " ", h.Group, a)
} }
v, ok := cl[pkg] if h.Fn != nil {
if v > high { for _, a := range h.Fn() {
high = v writeAttr(eb, " ", h.Group, a)
}
} }
if ok && v >= level { fmt.Fprint(eb, "\n")
return true, high } else {
var wrotePkgs bool
ensurePkgs := func() {
if !wrotePkgs {
wrotePkgs = true
for _, pkg := range h.Pkgs {
writeAttr(eb, "; ", "", slog.String("pkg", pkg))
}
}
} }
seen = seen || ok
fmt.Fprint(eb, LevelStrings[r.Level], ": ", r.Message)
n := 0
r.Attrs(func(a slog.Attr) bool {
if n == 0 && a.Key == "err" && h.Group == "" {
fmt.Fprint(eb, ": ", a.Value.String())
ensurePkgs()
} else {
ensurePkgs()
writeAttr(eb, "; ", h.Group, a)
}
n++
return true
})
ensurePkgs()
for _, a := range h.Attrs {
writeAttr(eb, "; ", h.Group, a)
}
if h.Fn != nil {
for _, a := range h.Fn() {
writeAttr(eb, "; ", h.Group, a)
n++
}
}
fmt.Fprint(eb, "\n")
} }
if seen { if eb.Err != nil {
return false, high return eb.Err
} }
v, ok := cl[""]
if v > high { // todo: for mox serve, do writes in separate goroutine.
high = v _, err := os.Stderr.Write(b.Bytes())
} return err
return ok && v >= level, v
} }
type errWriter struct { type errWriter struct {
log *Log Writer *bytes.Buffer
level Level Err error
msg string
} }
func (w *errWriter) Write(buf []byte) (int, error) { func (w *errWriter) Write(buf []byte) (int, error) {
err := errors.New(strings.TrimSpace(string(buf))) if w.Err != nil {
w.log.logx(w.level, err, w.msg) return 0, w.Err
}
var n int
n, w.Err = w.Writer.Write(buf)
return n, w.Err
}
func (h *handler) WithAttrs(attrs []slog.Attr) slog.Handler {
nh := *h
if h.Attrs != nil {
nh.Attrs = append([]slog.Attr{}, h.Attrs...)
}
nh.Attrs = append(nh.Attrs, attrs...)
return &nh
}
func (h *handler) WithGroup(name string) slog.Handler {
if name == "" {
return h
}
nh := *h
nh.Group += name + "."
return &nh
}
func (h *handler) WithPkg(pkg string) *handler {
nh := *h
if nh.Pkgs != nil {
nh.Pkgs = append([]string{}, nh.Pkgs...)
}
nh.Pkgs = append(nh.Pkgs, pkg)
return &nh
}
func (h *handler) WithFunc(fn func() []slog.Attr) *handler {
nh := *h
nh.Fn = fn
return &nh
}
type logWriter struct {
log Log
level slog.Level
msg string
}
func (w logWriter) Write(buf []byte) (int, error) {
err := strings.TrimSpace(string(buf))
w.log.LogAttrs(noctx, w.level, w.msg, slog.String("err", err))
return len(buf), nil return len(buf), nil
} }
// ErrWriter returns a writer that turns each write into a logging call on "log" // LogWriter returns a writer that turns each write into a logging call on "log"
// with given "level" and "msg" and the written content as an error. // with given "level" and "msg" and the written content as an error.
// Can be used for making a Go log.Logger for use in http.Server.ErrorLog. // Can be used for making a Go log.Logger for use in http.Server.ErrorLog.
func ErrWriter(log *Log, level Level, msg string) io.Writer { func LogWriter(log Log, level slog.Level, msg string) io.Writer {
return &errWriter{log, level, msg} return logWriter{log, level, msg}
} }

72
mox-/admin.go
View file

@ -20,6 +20,7 @@ import (
"time" "time"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/mjl-/adns" "github.com/mjl-/adns"
@ -28,7 +29,6 @@ import (
"github.com/mjl-/mox/dmarc" "github.com/mjl-/mox/dmarc"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/junk" "github.com/mjl-/mox/junk"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mtasts" "github.com/mjl-/mox/mtasts"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/tlsrpt" "github.com/mjl-/mox/tlsrpt"
@ -154,7 +154,7 @@ func MakeAccountConfig(addr smtp.Address) config.Account {
// MakeDomainConfig makes a new config for a domain, creating DKIM keys, using // MakeDomainConfig makes a new config for a domain, creating DKIM keys, using
// accountName for DMARC and TLS reports. // accountName for DMARC and TLS reports.
func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountName string, withMTASTS bool) (config.Domain, []string, error) { func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountName string, withMTASTS bool) (config.Domain, []string, error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
now := time.Now() now := time.Now()
year := now.Format("2006") year := now.Format("2006")
@ -164,7 +164,7 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN
defer func() { defer func() {
for _, p := range paths { for _, p := range paths {
err := os.Remove(p) err := os.Remove(p)
log.Check(err, "removing path for domain config", mlog.Field("path", p)) log.Check(err, "removing path for domain config", slog.String("path", p))
} }
}() }()
@ -180,7 +180,7 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN
err := f.Close() err := f.Close()
log.Check(err, "closing file after error") log.Check(err, "closing file after error")
err = os.Remove(path) err = os.Remove(path)
log.Check(err, "removing file after error", mlog.Field("path", path)) log.Check(err, "removing file after error", slog.String("path", path))
} }
}() }()
if _, err := f.Write(data); err != nil { if _, err := f.Write(data); err != nil {
@ -288,10 +288,10 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN
// If the account does not exist, it is created with localpart. Localpart must be // If the account does not exist, it is created with localpart. Localpart must be
// set only if the account does not yet exist. // set only if the account does not yet exist.
func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, localpart smtp.Localpart) (rerr error) { func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, localpart smtp.Localpart) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("adding domain", rerr, mlog.Field("domain", domain), mlog.Field("account", accountName), mlog.Field("localpart", localpart)) log.Errorx("adding domain", rerr, slog.Any("domain", domain), slog.String("account", accountName), slog.Any("localpart", localpart))
} }
}() }()
@ -327,7 +327,7 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local
defer func() { defer func() {
for _, f := range cleanupFiles { for _, f := range cleanupFiles {
err := os.Remove(f) err := os.Remove(f)
log.Check(err, "cleaning up file after error", mlog.Field("path", f)) log.Check(err, "cleaning up file after error", slog.String("path", f))
} }
}() }()
@ -356,7 +356,7 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("domain added", mlog.Field("domain", domain)) log.Info("domain added", slog.Any("domain", domain))
cleanupFiles = nil // All good, don't cleanup. cleanupFiles = nil // All good, don't cleanup.
return nil return nil
} }
@ -365,10 +365,10 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local
// //
// No accounts are removed, also not when they still reference this domain. // No accounts are removed, also not when they still reference this domain.
func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) { func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("removing domain", rerr, mlog.Field("domain", domain)) log.Errorx("removing domain", rerr, slog.Any("domain", domain))
} }
}() }()
@ -418,16 +418,16 @@ func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) {
err = os.Rename(src, dst) err = os.Rename(src, dst)
} }
if err != nil { if err != nil {
log.Errorx("renaming dkim private key file for removed domain", err, mlog.Field("src", src), mlog.Field("dst", dst)) log.Errorx("renaming dkim private key file for removed domain", err, slog.String("src", src), slog.String("dst", dst))
} }
} }
log.Info("domain removed", mlog.Field("domain", domain)) log.Info("domain removed", slog.Any("domain", domain))
return nil return nil
} }
func WebserverConfigSet(ctx context.Context, domainRedirects map[string]string, webhandlers []config.WebHandler) (rerr error) { func WebserverConfigSet(ctx context.Context, domainRedirects map[string]string, webhandlers []config.WebHandler) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("saving webserver config", rerr) log.Errorx("saving webserver config", rerr)
@ -680,10 +680,10 @@ func DomainRecords(domConf config.Domain, domain dns.Domain, hasDNSSEC bool) ([]
// //
// Catchall addresses are not supported for AccountAdd. Add separately with AddressAdd. // Catchall addresses are not supported for AccountAdd. Add separately with AddressAdd.
func AccountAdd(ctx context.Context, account, address string) (rerr error) { func AccountAdd(ctx context.Context, account, address string) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("adding account", rerr, mlog.Field("account", account), mlog.Field("address", address)) log.Errorx("adding account", rerr, slog.String("account", account), slog.String("address", address))
} }
}() }()
@ -716,16 +716,16 @@ func AccountAdd(ctx context.Context, account, address string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("account added", mlog.Field("account", account), mlog.Field("address", addr)) log.Info("account added", slog.String("account", account), slog.Any("address", addr))
return nil return nil
} }
// AccountRemove removes an account and reloads the configuration. // AccountRemove removes an account and reloads the configuration.
func AccountRemove(ctx context.Context, account string) (rerr error) { func AccountRemove(ctx context.Context, account string) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("adding account", rerr, mlog.Field("account", account)) log.Errorx("adding account", rerr, slog.String("account", account))
} }
}() }()
@ -750,7 +750,7 @@ func AccountRemove(ctx context.Context, account string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("account removed", mlog.Field("account", account)) log.Info("account removed", slog.String("account", account))
return nil return nil
} }
@ -775,10 +775,10 @@ func checkAddressAvailable(addr smtp.Address) error {
// AddressAdd adds an email address to an account and reloads the configuration. If // AddressAdd adds an email address to an account and reloads the configuration. If
// address starts with an @ it is treated as a catchall address for the domain. // address starts with an @ it is treated as a catchall address for the domain.
func AddressAdd(ctx context.Context, address, account string) (rerr error) { func AddressAdd(ctx context.Context, address, account string) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("adding address", rerr, mlog.Field("address", address), mlog.Field("account", account)) log.Errorx("adding address", rerr, slog.String("address", address), slog.String("account", account))
} }
}() }()
@ -834,16 +834,16 @@ func AddressAdd(ctx context.Context, address, account string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("address added", mlog.Field("address", address), mlog.Field("account", account)) log.Info("address added", slog.String("address", address), slog.String("account", account))
return nil return nil
} }
// AddressRemove removes an email address and reloads the configuration. // AddressRemove removes an email address and reloads the configuration.
func AddressRemove(ctx context.Context, address string) (rerr error) { func AddressRemove(ctx context.Context, address string) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("removing address", rerr, mlog.Field("address", address)) log.Errorx("removing address", rerr, slog.String("address", address))
} }
}() }()
@ -884,16 +884,16 @@ func AddressRemove(ctx context.Context, address string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("address removed", mlog.Field("address", address), mlog.Field("account", ad.Account)) log.Info("address removed", slog.String("address", address), slog.String("account", ad.Account))
return nil return nil
} }
// AccountFullNameSave updates the full name for an account and reloads the configuration. // AccountFullNameSave updates the full name for an account and reloads the configuration.
func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr error) { func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("saving account full name", rerr, mlog.Field("account", account)) log.Errorx("saving account full name", rerr, slog.String("account", account))
} }
}() }()
@ -920,16 +920,16 @@ func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr er
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("account full name saved", mlog.Field("account", account)) log.Info("account full name saved", slog.String("account", account))
return nil return nil
} }
// DestinationSave updates a destination for an account and reloads the configuration. // DestinationSave updates a destination for an account and reloads the configuration.
func DestinationSave(ctx context.Context, account, destName string, newDest config.Destination) (rerr error) { func DestinationSave(ctx context.Context, account, destName string, newDest config.Destination) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("saving destination", rerr, mlog.Field("account", account), mlog.Field("destname", destName), mlog.Field("destination", newDest)) log.Errorx("saving destination", rerr, slog.String("account", account), slog.String("destname", destName), slog.Any("destination", newDest))
} }
}() }()
@ -965,16 +965,16 @@ func DestinationSave(ctx context.Context, account, destName string, newDest conf
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("destination saved", mlog.Field("account", account), mlog.Field("destname", destName)) log.Info("destination saved", slog.String("account", account), slog.String("destname", destName))
return nil return nil
} }
// AccountLimitsSave saves new message sending limits for an account. // AccountLimitsSave saves new message sending limits for an account.
func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesPerDay, maxFirstTimeRecipientsPerDay int) (rerr error) { func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesPerDay, maxFirstTimeRecipientsPerDay int) (rerr error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
defer func() { defer func() {
if rerr != nil { if rerr != nil {
log.Errorx("saving account limits", rerr, mlog.Field("account", account)) log.Errorx("saving account limits", rerr, slog.String("account", account))
} }
}() }()
@ -1001,7 +1001,7 @@ func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesP
if err := writeDynamic(ctx, log, nc); err != nil { if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err) return fmt.Errorf("writing domains.conf: %v", err)
} }
log.Info("account limits saved", mlog.Field("account", account)) log.Info("account limits saved", slog.String("account", account))
return nil return nil
} }
@ -1157,7 +1157,7 @@ func ClientConfigsDomain(d dns.Domain) (ClientConfigs, error) {
// IPs returns ip addresses we may be listening/receiving mail on or // IPs returns ip addresses we may be listening/receiving mail on or
// connecting/sending from to the outside. // connecting/sending from to the outside.
func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) { func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) {
log := xlog.WithContext(ctx) log := pkglog.WithContext(ctx)
// Try to gather all IPs we are listening on by going through the config. // Try to gather all IPs we are listening on by going through the config.
// If we encounter 0.0.0.0 or ::, we'll gather all local IPs afterwards. // If we encounter 0.0.0.0 or ::, we'll gather all local IPs afterwards.
@ -1208,7 +1208,7 @@ func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) {
for _, addr := range addrs { for _, addr := range addrs {
ip, _, err := net.ParseCIDR(addr.String()) ip, _, err := net.ParseCIDR(addr.String())
if err != nil { if err != nil {
log.Errorx("bad interface addr", err, mlog.Field("address", addr)) log.Errorx("bad interface addr", err, slog.Any("address", addr))
continue continue
} }
v4 := ip.To4() != nil v4 := ip.To4() != nil

95
mox-/config.go
View file

@ -28,6 +28,7 @@ import (
"sync" "sync"
"time" "time"
"golang.org/x/exp/slog"
"golang.org/x/text/unicode/norm" "golang.org/x/text/unicode/norm"
"github.com/mjl-/autocert" "github.com/mjl-/autocert"
@ -44,14 +45,14 @@ import (
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
var xlog = mlog.New("mox") var pkglog = mlog.New("mox", nil)
// Config paths are set early in program startup. They will point to files in // Config paths are set early in program startup. They will point to files in
// the same directory. // the same directory.
var ( var (
ConfigStaticPath string ConfigStaticPath string
ConfigDynamicPath string ConfigDynamicPath string
Conf = Config{Log: map[string]mlog.Level{"": mlog.LevelError}} Conf = Config{Log: map[string]slog.Level{"": slog.LevelError}}
) )
// Config as used in the code, a processed version of what is in the config file. // Config as used in the code, a processed version of what is in the config file.
@ -61,7 +62,7 @@ type Config struct {
Static config.Static // Does not change during the lifetime of a running instance. Static config.Static // Does not change during the lifetime of a running instance.
logMutex sync.Mutex // For accessing the log levels. logMutex sync.Mutex // For accessing the log levels.
Log map[string]mlog.Level Log map[string]slog.Level
dynamicMutex sync.Mutex dynamicMutex sync.Mutex
Dynamic config.Dynamic // Can only be accessed directly by tests. Use methods on Config for locked access. Dynamic config.Dynamic // Can only be accessed directly by tests. Use methods on Config for locked access.
@ -83,31 +84,31 @@ type AccountDestination struct {
// LogLevelSet sets a new log level for pkg. An empty pkg sets the default log // LogLevelSet sets a new log level for pkg. An empty pkg sets the default log
// value that is used if no explicit log level is configured for a package. // value that is used if no explicit log level is configured for a package.
// This change is ephemeral, no config file is changed. // This change is ephemeral, no config file is changed.
func (c *Config) LogLevelSet(pkg string, level mlog.Level) { func (c *Config) LogLevelSet(log mlog.Log, pkg string, level slog.Level) {
c.logMutex.Lock() c.logMutex.Lock()
defer c.logMutex.Unlock() defer c.logMutex.Unlock()
l := c.copyLogLevels() l := c.copyLogLevels()
l[pkg] = level l[pkg] = level
c.Log = l c.Log = l
xlog.Print("log level changed", mlog.Field("pkg", pkg), mlog.Field("level", mlog.LevelStrings[level])) log.Print("log level changed", slog.String("pkg", pkg), slog.Any("level", mlog.LevelStrings[level]))
mlog.SetConfig(c.Log) mlog.SetConfig(c.Log)
} }
// LogLevelRemove removes a configured log level for a package. // LogLevelRemove removes a configured log level for a package.
func (c *Config) LogLevelRemove(pkg string) { func (c *Config) LogLevelRemove(log mlog.Log, pkg string) {
c.logMutex.Lock() c.logMutex.Lock()
defer c.logMutex.Unlock() defer c.logMutex.Unlock()
l := c.copyLogLevels() l := c.copyLogLevels()
delete(l, pkg) delete(l, pkg)
c.Log = l c.Log = l
xlog.Print("log level cleared", mlog.Field("pkg", pkg)) log.Print("log level cleared", slog.String("pkg", pkg))
mlog.SetConfig(c.Log) mlog.SetConfig(c.Log)
} }
// copyLogLevels returns a copy of c.Log, for modifications. // copyLogLevels returns a copy of c.Log, for modifications.
// must be called with log lock held. // must be called with log lock held.
func (c *Config) copyLogLevels() map[string]mlog.Level { func (c *Config) copyLogLevels() map[string]slog.Level {
m := map[string]mlog.Level{} m := map[string]slog.Level{}
for pkg, level := range c.Log { for pkg, level := range c.Log {
m[pkg] = level m[pkg] = level
} }
@ -115,7 +116,7 @@ func (c *Config) copyLogLevels() map[string]mlog.Level {
} }
// LogLevels returns a copy of the current log levels. // LogLevels returns a copy of the current log levels.
func (c *Config) LogLevels() map[string]mlog.Level { func (c *Config) LogLevels() map[string]slog.Level {
c.logMutex.Lock() c.logMutex.Lock()
defer c.logMutex.Unlock() defer c.logMutex.Unlock()
return c.copyLogLevels() return c.copyLogLevels()
@ -128,12 +129,12 @@ func (c *Config) withDynamicLock(fn func()) {
if now.Sub(c.DynamicLastCheck) > time.Second { if now.Sub(c.DynamicLastCheck) > time.Second {
c.DynamicLastCheck = now c.DynamicLastCheck = now
if fi, err := os.Stat(ConfigDynamicPath); err != nil { if fi, err := os.Stat(ConfigDynamicPath); err != nil {
xlog.Errorx("stat domains config", err) pkglog.Errorx("stat domains config", err)
} else if !fi.ModTime().Equal(c.dynamicMtime) { } else if !fi.ModTime().Equal(c.dynamicMtime) {
if errs := c.loadDynamic(); len(errs) > 0 { if errs := c.loadDynamic(); len(errs) > 0 {
xlog.Errorx("loading domains config", errs[0], mlog.Field("errors", errs)) pkglog.Errorx("loading domains config", errs[0], slog.Any("errors", errs))
} else { } else {
xlog.Info("domains config reloaded") pkglog.Info("domains config reloaded")
c.dynamicMtime = fi.ModTime() c.dynamicMtime = fi.ModTime()
} }
} }
@ -143,14 +144,14 @@ func (c *Config) withDynamicLock(fn func()) {
// must be called with dynamic lock held. // must be called with dynamic lock held.
func (c *Config) loadDynamic() []error { func (c *Config) loadDynamic() []error {
d, mtime, accDests, err := ParseDynamicConfig(context.Background(), ConfigDynamicPath, c.Static) d, mtime, accDests, err := ParseDynamicConfig(context.Background(), pkglog, ConfigDynamicPath, c.Static)
if err != nil { if err != nil {
return err return err
} }
c.Dynamic = d c.Dynamic = d
c.dynamicMtime = mtime c.dynamicMtime = mtime
c.accountDestinations = accDests c.accountDestinations = accDests
c.allowACMEHosts(true) c.allowACMEHosts(pkglog, true)
return nil return nil
} }
@ -236,7 +237,7 @@ func (c *Config) Routes(accountName string, domain dns.Domain) (accountRoutes, d
return return
} }
func (c *Config) allowACMEHosts(checkACMEHosts bool) { func (c *Config) allowACMEHosts(log mlog.Log, checkACMEHosts bool) {
for _, l := range c.Static.Listeners { for _, l := range c.Static.Listeners {
if l.TLS == nil || l.TLS.ACME == "" { if l.TLS == nil || l.TLS.ACME == "" {
continue continue
@ -259,7 +260,7 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) {
if l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS { if l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS {
if d, err := dns.ParseDomain("autoconfig." + dom.Domain.ASCII); err != nil { if d, err := dns.ParseDomain("autoconfig." + dom.Domain.ASCII); err != nil {
xlog.Errorx("parsing autoconfig domain", err, mlog.Field("domain", dom.Domain)) log.Errorx("parsing autoconfig domain", err, slog.Any("domain", dom.Domain))
} else { } else {
hostnames[d] = struct{}{} hostnames[d] = struct{}{}
} }
@ -268,7 +269,7 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) {
if l.MTASTSHTTPS.Enabled && dom.MTASTS != nil && !l.MTASTSHTTPS.NonTLS { if l.MTASTSHTTPS.Enabled && dom.MTASTS != nil && !l.MTASTSHTTPS.NonTLS {
d, err := dns.ParseDomain("mta-sts." + dom.Domain.ASCII) d, err := dns.ParseDomain("mta-sts." + dom.Domain.ASCII)
if err != nil { if err != nil {
xlog.Errorx("parsing mta-sts domain", err, mlog.Field("domain", dom.Domain)) log.Errorx("parsing mta-sts domain", err, slog.Any("domain", dom.Domain))
} else { } else {
hostnames[d] = struct{}{} hostnames[d] = struct{}{}
} }
@ -292,15 +293,15 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) {
if public.IPsNATed { if public.IPsNATed {
ips = nil ips = nil
} }
m.SetAllowedHostnames(dns.StrictResolver{Pkg: "autotls"}, hostnames, ips, checkACMEHosts) m.SetAllowedHostnames(log, dns.StrictResolver{Pkg: "autotls", Log: log.Logger}, hostnames, ips, checkACMEHosts)
} }
} }
// todo future: write config parsing & writing code that can read a config and remembers the exact tokens including newlines and comments, and can write back a modified file. the goal is to be able to write a config file automatically (after changing fields through the ui), but not loose comments and whitespace, to still get useful diffs for storing the config in a version control system. // todo future: write config parsing & writing code that can read a config and remembers the exact tokens including newlines and comments, and can write back a modified file. the goal is to be able to write a config file automatically (after changing fields through the ui), but not loose comments and whitespace, to still get useful diffs for storing the config in a version control system.
// must be called with lock held. // must be called with lock held.
func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error { func writeDynamic(ctx context.Context, log mlog.Log, c config.Dynamic) error {
accDests, errs := prepareDynamicConfig(ctx, ConfigDynamicPath, Conf.Static, &c) accDests, errs := prepareDynamicConfig(ctx, log, ConfigDynamicPath, Conf.Static, &c)
if len(errs) > 0 { if len(errs) > 0 {
return errs[0] return errs[0]
} }
@ -330,7 +331,7 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error {
if err := f.Sync(); err != nil { if err := f.Sync(); err != nil {
return fmt.Errorf("sync domains.conf after write: %v", err) return fmt.Errorf("sync domains.conf after write: %v", err)
} }
if err := moxio.SyncDir(filepath.Dir(ConfigDynamicPath)); err != nil { if err := moxio.SyncDir(log, filepath.Dir(ConfigDynamicPath)); err != nil {
return fmt.Errorf("sync dir of domains.conf after write: %v", err) return fmt.Errorf("sync dir of domains.conf after write: %v", err)
} }
@ -349,32 +350,32 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error {
Conf.Dynamic = c Conf.Dynamic = c
Conf.accountDestinations = accDests Conf.accountDestinations = accDests
Conf.allowACMEHosts(true) Conf.allowACMEHosts(log, true)
return nil return nil
} }
// MustLoadConfig loads the config, quitting on errors. // MustLoadConfig loads the config, quitting on errors.
func MustLoadConfig(doLoadTLSKeyCerts, checkACMEHosts bool) { func MustLoadConfig(doLoadTLSKeyCerts, checkACMEHosts bool) {
errs := LoadConfig(context.Background(), doLoadTLSKeyCerts, checkACMEHosts) errs := LoadConfig(context.Background(), pkglog, doLoadTLSKeyCerts, checkACMEHosts)
if len(errs) > 1 { if len(errs) > 1 {
xlog.Error("loading config file: multiple errors") pkglog.Error("loading config file: multiple errors")
for _, err := range errs { for _, err := range errs {
xlog.Errorx("config error", err) pkglog.Errorx("config error", err)
} }
xlog.Fatal("stopping after multiple config errors") pkglog.Fatal("stopping after multiple config errors")
} else if len(errs) == 1 { } else if len(errs) == 1 {
xlog.Fatalx("loading config file", errs[0]) pkglog.Fatalx("loading config file", errs[0])
} }
} }
// LoadConfig attempts to parse and load a config, returning any errors // LoadConfig attempts to parse and load a config, returning any errors
// encountered. // encountered.
func LoadConfig(ctx context.Context, doLoadTLSKeyCerts, checkACMEHosts bool) []error { func LoadConfig(ctx context.Context, log mlog.Log, doLoadTLSKeyCerts, checkACMEHosts bool) []error {
Shutdown, ShutdownCancel = context.WithCancel(context.Background()) Shutdown, ShutdownCancel = context.WithCancel(context.Background())
Context, ContextCancel = context.WithCancel(context.Background()) Context, ContextCancel = context.WithCancel(context.Background())
c, errs := ParseConfig(ctx, ConfigStaticPath, false, doLoadTLSKeyCerts, checkACMEHosts) c, errs := ParseConfig(ctx, log, ConfigStaticPath, false, doLoadTLSKeyCerts, checkACMEHosts)
if len(errs) > 0 { if len(errs) > 0 {
return errs return errs
} }
@ -405,7 +406,7 @@ func SetConfig(c *Config) {
// quickstart in the case the user is going to provide their own certificates. // quickstart in the case the user is going to provide their own certificates.
// If checkACMEHosts is true, the hosts allowed for acme are compared with the // If checkACMEHosts is true, the hosts allowed for acme are compared with the
// explicitly configured ips we are listening on. // explicitly configured ips we are listening on.
func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, checkACMEHosts bool) (c *Config, errs []error) { func ParseConfig(ctx context.Context, log mlog.Log, p string, checkOnly, doLoadTLSKeyCerts, checkACMEHosts bool) (c *Config, errs []error) {
c = &Config{ c = &Config{
Static: config.Static{ Static: config.Static{
DataDir: ".", DataDir: ".",
@ -424,15 +425,15 @@ func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, ch
return nil, []error{fmt.Errorf("parsing %s%v", p, err)} return nil, []error{fmt.Errorf("parsing %s%v", p, err)}
} }
if xerrs := PrepareStaticConfig(ctx, p, c, checkOnly, doLoadTLSKeyCerts); len(xerrs) > 0 { if xerrs := PrepareStaticConfig(ctx, log, p, c, checkOnly, doLoadTLSKeyCerts); len(xerrs) > 0 {
return nil, xerrs return nil, xerrs
} }
pp := filepath.Join(filepath.Dir(p), "domains.conf") pp := filepath.Join(filepath.Dir(p), "domains.conf")
c.Dynamic, c.dynamicMtime, c.accountDestinations, errs = ParseDynamicConfig(ctx, pp, c.Static) c.Dynamic, c.dynamicMtime, c.accountDestinations, errs = ParseDynamicConfig(ctx, log, pp, c.Static)
if !checkOnly { if !checkOnly {
c.allowACMEHosts(checkACMEHosts) c.allowACMEHosts(log, checkACMEHosts)
} }
return c, errs return c, errs
@ -441,13 +442,11 @@ func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, ch
// PrepareStaticConfig parses the static config file and prepares data structures // PrepareStaticConfig parses the static config file and prepares data structures
// for starting mox. If checkOnly is set no substantial changes are made, like // for starting mox. If checkOnly is set no substantial changes are made, like
// creating an ACME registration. // creating an ACME registration.
func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, checkOnly, doLoadTLSKeyCerts bool) (errs []error) { func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, conf *Config, checkOnly, doLoadTLSKeyCerts bool) (errs []error) {
addErrorf := func(format string, args ...any) { addErrorf := func(format string, args ...any) {
errs = append(errs, fmt.Errorf(format, args...)) errs = append(errs, fmt.Errorf(format, args...))
} }
log := xlog.WithContext(ctx)
c := &conf.Static c := &conf.Static
// check that mailbox is in unicode NFC normalized form. // check that mailbox is in unicode NFC normalized form.
@ -461,7 +460,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
// Post-process logging config. // Post-process logging config.
if logLevel, ok := mlog.Levels[c.LogLevel]; ok { if logLevel, ok := mlog.Levels[c.LogLevel]; ok {
conf.Log = map[string]mlog.Level{"": logLevel} conf.Log = map[string]slog.Level{"": logLevel}
} else { } else {
addErrorf("invalid log level %q", c.LogLevel) addErrorf("invalid log level %q", c.LogLevel)
} }
@ -569,10 +568,10 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
key = findACMEHostPrivateKey(acmeName, host, keyType, 2) key = findACMEHostPrivateKey(acmeName, host, keyType, 2)
} }
if key != nil { if key != nil {
log.Debug("found existing private key for certificate for host", mlog.Field("acmename", acmeName), mlog.Field("host", host), mlog.Field("keytype", keyType)) log.Debug("found existing private key for certificate for host", slog.String("acmename", acmeName), slog.String("host", host), slog.Any("keytype", keyType))
return key, nil return key, nil
} }
log.Debug("generating new private key for certificate for host", mlog.Field("acmename", acmeName), mlog.Field("host", host), mlog.Field("keytype", keyType)) log.Debug("generating new private key for certificate for host", slog.String("acmename", acmeName), slog.String("host", host), slog.Any("keytype", keyType))
switch keyType { switch keyType {
case autocert.KeyRSA2048: case autocert.KeyRSA2048:
return rsa.GenerateKey(cryptorand.Reader, 2048) return rsa.GenerateKey(cryptorand.Reader, 2048)
@ -658,18 +657,18 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
switch k := privKey.(type) { switch k := privKey.(type) {
case *rsa.PrivateKey: case *rsa.PrivateKey:
if k.N.BitLen() != 2048 { if k.N.BitLen() != 2048 {
log.Error("need rsa key with 2048 bits, for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath), mlog.Field("bits", k.N.BitLen())) log.Error("need rsa key with 2048 bits, for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath), slog.Int("bits", k.N.BitLen()))
continue continue
} }
l.TLS.HostPrivateRSA2048Keys = append(l.TLS.HostPrivateRSA2048Keys, k) l.TLS.HostPrivateRSA2048Keys = append(l.TLS.HostPrivateRSA2048Keys, k)
case *ecdsa.PrivateKey: case *ecdsa.PrivateKey:
if k.Curve != elliptic.P256() { if k.Curve != elliptic.P256() {
log.Error("unrecognized ecdsa curve for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath)) log.Error("unrecognized ecdsa curve for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath))
continue continue
} }
l.TLS.HostPrivateECDSAP256Keys = append(l.TLS.HostPrivateECDSAP256Keys, k) l.TLS.HostPrivateECDSAP256Keys = append(l.TLS.HostPrivateECDSAP256Keys, k)
default: default:
log.Error("unrecognized key type for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath), mlog.Field("keytype", fmt.Sprintf("%T", privKey))) log.Error("unrecognized key type for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath), slog.String("keytype", fmt.Sprintf("%T", privKey)))
continue continue
} }
} }
@ -914,7 +913,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
} }
// PrepareDynamicConfig parses the dynamic config file given a static file. // PrepareDynamicConfig parses the dynamic config file given a static file.
func ParseDynamicConfig(ctx context.Context, dynamicPath string, static config.Static) (c config.Dynamic, mtime time.Time, accDests map[string]AccountDestination, errs []error) { func ParseDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, static config.Static) (c config.Dynamic, mtime time.Time, accDests map[string]AccountDestination, errs []error) {
addErrorf := func(format string, args ...any) { addErrorf := func(format string, args ...any) {
errs = append(errs, fmt.Errorf(format, args...)) errs = append(errs, fmt.Errorf(format, args...))
} }
@ -934,13 +933,11 @@ func ParseDynamicConfig(ctx context.Context, dynamicPath string, static config.S
return return
} }
accDests, errs = prepareDynamicConfig(ctx, dynamicPath, static, &c) accDests, errs = prepareDynamicConfig(ctx, log, dynamicPath, static, &c)
return c, fi.ModTime(), accDests, errs return c, fi.ModTime(), accDests, errs
} }
func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config.Static, c *config.Dynamic) (accDests map[string]AccountDestination, errs []error) { func prepareDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, static config.Static, c *config.Dynamic) (accDests map[string]AccountDestination, errs []error) {
log := xlog.WithContext(ctx)
addErrorf := func(format string, args ...any) { addErrorf := func(format string, args ...any) {
errs = append(errs, fmt.Errorf(format, args...)) errs = append(errs, fmt.Errorf(format, args...))
} }
@ -1321,7 +1318,7 @@ func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config
if !ok { if !ok {
addErrorf("could not find localpart %q to replace with address in destinations", lp) addErrorf("could not find localpart %q to replace with address in destinations", lp)
} else { } else {
log.Error(`deprecation warning: support for account destination addresses specified as just localpart ("username") instead of full email address will be removed in the future; update domains.conf, for each Account, for each Destination, ensure each key is an email address by appending "@" and the default domain for the account`, mlog.Field("localpart", lp), mlog.Field("address", addr), mlog.Field("account", accName)) log.Error(`deprecation warning: support for account destination addresses specified as just localpart ("username") instead of full email address will be removed in the future; update domains.conf, for each Account, for each Destination, ensure each key is an email address by appending "@" and the default domain for the account`, slog.Any("localpart", lp), slog.Any("address", addr), slog.String("account", accName))
acc.Destinations[addr] = dest acc.Destinations[addr] = dest
delete(acc.Destinations, lp) delete(acc.Destinations, lp)
} }

10
mox-/forkexec_unix.go
View file

@ -8,7 +8,7 @@ import (
"strings" "strings"
"syscall" "syscall"
"github.com/mjl-/mox/mlog" "golang.org/x/exp/slog"
) )
// Fork and exec as unprivileged user. // Fork and exec as unprivileged user.
@ -19,7 +19,7 @@ import (
func ForkExecUnprivileged() { func ForkExecUnprivileged() {
prog, err := os.Executable() prog, err := os.Executable()
if err != nil { if err != nil {
xlog.Fatalx("finding executable for exec", err) pkglog.Fatalx("finding executable for exec", err)
} }
files := []*os.File{os.Stdin, os.Stdout, os.Stderr} files := []*os.File{os.Stdin, os.Stdout, os.Stderr}
@ -49,7 +49,7 @@ func ForkExecUnprivileged() {
}, },
}) })
if err != nil { if err != nil {
xlog.Fatalx("fork and exec", err) pkglog.Fatalx("fork and exec", err)
} }
CleanupPassedFiles() CleanupPassedFiles()
@ -66,9 +66,9 @@ func ForkExecUnprivileged() {
st, err := p.Wait() st, err := p.Wait()
if err != nil { if err != nil {
xlog.Fatalx("wait", err) pkglog.Fatalx("wait", err)
} }
code := st.ExitCode() code := st.ExitCode()
xlog.Print("stopping after child exit", mlog.Field("exitcode", code)) pkglog.Print("stopping after child exit", slog.Int("exitcode", code))
os.Exit(code) os.Exit(code)
} }

10
mox-/lifecycle.go
View file

@ -32,7 +32,7 @@ func RestorePassedFiles() {
if runtime.GOOS == "linux" { if runtime.GOOS == "linux" {
linuxhint = " If you updated from v0.0.1, update the mox.service file to start as root (privileges are dropped): ./mox config printservice >mox.service && sudo systemctl daemon-reload && sudo systemctl restart mox." linuxhint = " If you updated from v0.0.1, update the mox.service file to start as root (privileges are dropped): ./mox config printservice >mox.service && sudo systemctl daemon-reload && sudo systemctl restart mox."
} }
xlog.Fatal("mox must be started as root, and will drop privileges after binding required sockets (missing environment variable MOX_SOCKETS)." + linuxhint) pkglog.Fatal("mox must be started as root, and will drop privileges after binding required sockets (missing environment variable MOX_SOCKETS)." + linuxhint)
} }
// 0,1,2 are stdin,stdout,stderr, 3 is the first passed fd (first listeners, then files). // 0,1,2 are stdin,stdout,stderr, 3 is the first passed fd (first listeners, then files).
@ -59,12 +59,12 @@ func RestorePassedFiles() {
func CleanupPassedFiles() { func CleanupPassedFiles() {
for _, f := range passedListeners { for _, f := range passedListeners {
err := f.Close() err := f.Close()
xlog.Check(err, "closing listener socket file descriptor") pkglog.Check(err, "closing listener socket file descriptor")
} }
for _, fl := range passedFiles { for _, fl := range passedFiles {
for _, f := range fl { for _, f := range fl {
err := f.Close() err := f.Close()
xlog.Check(err, "closing path file descriptor") pkglog.Check(err, "closing path file descriptor")
} }
} }
} }
@ -193,7 +193,7 @@ func (c *connections) Register(nc net.Conn, protocol, listener string) {
// doesn't hurt to log it. // doesn't hurt to log it.
select { select {
case <-Shutdown.Done(): case <-Shutdown.Done():
xlog.Error("new connection added while shutting down") pkglog.Error("new connection added while shutting down")
debug.PrintStack() debug.PrintStack()
default: default:
} }
@ -258,7 +258,7 @@ func (c *connections) Shutdown() {
defer c.Unlock() defer c.Unlock()
for nc := range c.conns { for nc := range c.conns {
if err := nc.SetDeadline(now); err != nil { if err := nc.SetDeadline(now); err != nil {
xlog.Errorx("setting immediate read/write deadline for shutdown", err) pkglog.Errorx("setting immediate read/write deadline for shutdown", err)
} }
} }
} }

8
message/tlsrecv.go → mox-/tlsrecv.go
View file

@ -1,14 +1,16 @@
package message package mox
import ( import (
"crypto/tls" "crypto/tls"
"fmt" "fmt"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
// TLSReceivedComment returns a comment about TLS of the connection for use in a Receive header. // TLSReceivedComment returns a comment about TLS of the connection for use in a Receive header.
func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string { func TLSReceivedComment(log mlog.Log, cs tls.ConnectionState) []string {
// todo future: we could use the "tls" clause for the Received header as specified in ../rfc/8314:496. however, the text implies it is only for submission, not regular smtp. and it cannot specify the tls version. for now, not worth the trouble. // todo future: we could use the "tls" clause for the Received header as specified in ../rfc/8314:496. however, the text implies it is only for submission, not regular smtp. and it cannot specify the tls version. for now, not worth the trouble.
// Comments from other mail servers: // Comments from other mail servers:
@ -32,7 +34,7 @@ func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string {
if version, ok := versions[cs.Version]; ok { if version, ok := versions[cs.Version]; ok {
add(version) add(version)
} else { } else {
log.Info("unknown tls version identifier", mlog.Field("version", cs.Version)) log.Info("unknown tls version identifier", slog.Any("version", cs.Version))
add(fmt.Sprintf("TLS identifier %x", cs.Version)) add(fmt.Sprintf("TLS identifier %x", cs.Version))
} }

12
moxio/bufpool.go
View file

@ -6,11 +6,11 @@ import (
"fmt" "fmt"
"io" "io"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
var xlog = mlog.New("moxio")
// todo: instead of a bufpool, should maybe just make an alternative to bufio.Reader with a big enough buffer that we can fully use to read a line. // todo: instead of a bufpool, should maybe just make an alternative to bufio.Reader with a big enough buffer that we can fully use to read a line.
var ErrLineTooLong = errors.New("line from remote too long") // Returned by Bufpool.Readline. var ErrLineTooLong = errors.New("line from remote too long") // Returned by Bufpool.Readline.
@ -49,9 +49,9 @@ func (b *Bufpool) get() []byte {
// be all the bytes that have been read in the buffer. If the pool is full, the // be all the bytes that have been read in the buffer. If the pool is full, the
// buffer is discarded, and will be cleaned up by the garbage collector. // buffer is discarded, and will be cleaned up by the garbage collector.
// The caller should no longer reference "buf" after a call to put. // The caller should no longer reference "buf" after a call to put.
func (b *Bufpool) put(buf []byte, n int) { func (b *Bufpool) put(log mlog.Log, buf []byte, n int) {
if len(buf) != b.size { if len(buf) != b.size {
xlog.Error("buffer with bad size returned, ignoring", mlog.Field("badsize", len(buf)), mlog.Field("expsize", b.size)) log.Error("buffer with bad size returned, ignoring", slog.Int("badsize", len(buf)), slog.Int("expsize", b.size))
return return
} }
@ -67,11 +67,11 @@ func (b *Bufpool) put(buf []byte, n int) {
// Readline reads a \n- or \r\n-terminated line. Line is returned without \n or \r\n. // Readline reads a \n- or \r\n-terminated line. Line is returned without \n or \r\n.
// If the line was too long, ErrLineTooLong is returned. // If the line was too long, ErrLineTooLong is returned.
// If an EOF is encountered before a \n, io.ErrUnexpectedEOF is returned. // If an EOF is encountered before a \n, io.ErrUnexpectedEOF is returned.
func (b *Bufpool) Readline(r *bufio.Reader) (line string, rerr error) { func (b *Bufpool) Readline(log mlog.Log, r *bufio.Reader) (line string, rerr error) {
var nread int var nread int
buf := b.get() buf := b.get()
defer func() { defer func() {
b.put(buf, nread) b.put(log, buf, nread)
}() }()
// Read until newline. If we reach the end of the buffer first, we write back an // Read until newline. If we reach the end of the buffer first, we write back an

17
moxio/bufpool_test.go
View file

@ -7,6 +7,8 @@ import (
"io" "io"
"strings" "strings"
"testing" "testing"
"github.com/mjl-/mox/mlog"
) )
func TestBufpool(t *testing.T) { func TestBufpool(t *testing.T) {
@ -16,8 +18,9 @@ func TestBufpool(t *testing.T) {
for i := 0; i < len(a); i++ { for i := 0; i < len(a); i++ {
a[i] = 1 a[i] = 1
} }
bp.put(a, len(a)) // Will be stored. log := mlog.New("moxio", nil)
bp.put(b, 0) // Will be discarded. bp.put(log, a, len(a)) // Will be stored.
bp.put(log, b, 0) // Will be discarded.
na := bp.get() na := bp.get()
if fmt.Sprintf("%p", a) != fmt.Sprintf("%p", na) { if fmt.Sprintf("%p", a) != fmt.Sprintf("%p", na) {
t.Fatalf("received unexpected new buf %p != %p", a, na) t.Fatalf("received unexpected new buf %p != %p", a, na)
@ -28,22 +31,22 @@ func TestBufpool(t *testing.T) {
} }
} }
if _, err := bp.Readline(bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) { if _, err := bp.Readline(log, bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) {
t.Fatalf("expected ErrLineTooLong, got error %v", err) t.Fatalf("expected ErrLineTooLong, got error %v", err)
} }
if _, err := bp.Readline(bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) { if _, err := bp.Readline(log, bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) {
t.Fatalf("expected ErrLineTooLong, got error %v", err) t.Fatalf("expected ErrLineTooLong, got error %v", err)
} }
er := errReader{fmt.Errorf("bad")} er := errReader{fmt.Errorf("bad")}
if _, err := bp.Readline(bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) { if _, err := bp.Readline(log, bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) {
t.Fatalf("got unexpected error %s", err) t.Fatalf("got unexpected error %s", err)
} }
if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" { if line, err := bp.Readline(log, bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" {
t.Fatalf(`got %q, err %v, expected line "ok"`, line, err) t.Fatalf(`got %q, err %v, expected line "ok"`, line, err)
} }
if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" { if line, err := bp.Readline(log, bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" {
t.Fatalf(`got %q, err %v, expected line "ok"`, line, err) t.Fatalf(`got %q, err %v, expected line "ok"`, line, err)
} }
} }

8
moxio/linkcopy.go
View file

@ -5,6 +5,8 @@ import (
"io" "io"
"os" "os"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
@ -14,7 +16,7 @@ import (
// ensure the file is written on disk. Callers should also sync the directory of // ensure the file is written on disk. Callers should also sync the directory of
// the destination file, but may want to do that after linking/copying multiple // the destination file, but may want to do that after linking/copying multiple
// files. If dst was created and an error occurred, it is removed. // files. If dst was created and an error occurred, it is removed.
func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync bool) (rerr error) { func LinkOrCopy(log mlog.Log, dst, src string, srcReaderOpt io.Reader, sync bool) (rerr error) {
// Try hardlink first. // Try hardlink first.
err := os.Link(src, dst) err := os.Link(src, dst)
if err == nil { if err == nil {
@ -48,7 +50,7 @@ func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync boo
err := df.Close() err := df.Close()
log.Check(err, "closing partial destination file") log.Check(err, "closing partial destination file")
err = os.Remove(dst) err = os.Remove(dst)
log.Check(err, "removing partial destination file", mlog.Field("path", dst)) log.Check(err, "removing partial destination file", slog.String("path", dst))
} }
}() }()
@ -64,7 +66,7 @@ func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync boo
df = nil df = nil
if err != nil { if err != nil {
err := os.Remove(dst) err := os.Remove(dst)
log.Check(err, "removing partial destination file", mlog.Field("path", dst)) log.Check(err, "removing partial destination file", slog.String("path", dst))
return err return err
} }
return nil return nil

2
moxio/linkcopy_test.go
View file

@ -17,7 +17,7 @@ func tcheckf(t *testing.T, err error, format string, args ...any) {
} }
func TestLinkOrCopy(t *testing.T) { func TestLinkOrCopy(t *testing.T) {
log := mlog.New("linkorcopy") log := mlog.New("linkorcopy", nil)
// link in same directory. file exists error. link to file in non-existent // link in same directory. file exists error. link to file in non-existent
// directory (exists error). link to file in system temp dir (hopefully other file // directory (exists error). link to file in system temp dir (hopefully other file

6
moxio/syncdir.go
View file

@ -5,16 +5,18 @@ package moxio
import ( import (
"fmt" "fmt"
"os" "os"
"github.com/mjl-/mox/mlog"
) )
// SyncDir opens a directory and syncs its contents to disk. // SyncDir opens a directory and syncs its contents to disk.
func SyncDir(dir string) error { func SyncDir(log mlog.Log, dir string) error {
d, err := os.Open(dir) d, err := os.Open(dir)
if err != nil { if err != nil {
return fmt.Errorf("open directory: %v", err) return fmt.Errorf("open directory: %v", err)
} }
err = d.Sync() err = d.Sync()
xerr := d.Close() xerr := d.Close()
xlog.Check(xerr, "closing directory after sync") log.Check(xerr, "closing directory after sync")
return err return err
} }

22
moxio/trace.go
View file

@ -3,43 +3,45 @@ package moxio
import ( import (
"io" "io"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
type TraceWriter struct { type TraceWriter struct {
log *mlog.Log log mlog.Log
prefix string prefix string
w io.Writer w io.Writer
level mlog.Level level slog.Level
} }
// NewTraceWriter wraps "w" into a writer that logs all writes to "log" with // NewTraceWriter wraps "w" into a writer that logs all writes to "log" with
// log level trace, prefixed with "prefix". // log level trace, prefixed with "prefix".
func NewTraceWriter(log *mlog.Log, prefix string, w io.Writer) *TraceWriter { func NewTraceWriter(log mlog.Log, prefix string, w io.Writer) *TraceWriter {
return &TraceWriter{log, prefix, w, mlog.LevelTrace} return &TraceWriter{log, prefix, w, mlog.LevelTrace}
} }
// Write logs a trace line for writing buf to the client, then writes to the // Write logs a trace line for writing buf to the client, then writes to the
// client. // client.
func (w *TraceWriter) Write(buf []byte) (int, error) { func (w *TraceWriter) Write(buf []byte) (int, error) {
w.log.Trace(w.level, w.prefix+string(buf)) w.log.Trace(w.level, w.prefix, buf)
return w.w.Write(buf) return w.w.Write(buf)
} }
func (w *TraceWriter) SetTrace(level mlog.Level) { func (w *TraceWriter) SetTrace(level slog.Level) {
w.level = level w.level = level
} }
type TraceReader struct { type TraceReader struct {
log *mlog.Log log mlog.Log
prefix string prefix string
r io.Reader r io.Reader
level mlog.Level level slog.Level
} }
// NewTraceReader wraps reader "r" into a reader that logs all reads to "log" // NewTraceReader wraps reader "r" into a reader that logs all reads to "log"
// with log level trace, prefixed with "prefix". // with log level trace, prefixed with "prefix".
func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) *TraceReader { func NewTraceReader(log mlog.Log, prefix string, r io.Reader) *TraceReader {
return &TraceReader{log, prefix, r, mlog.LevelTrace} return &TraceReader{log, prefix, r, mlog.LevelTrace}
} }
@ -48,11 +50,11 @@ func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) *TraceReader {
func (r *TraceReader) Read(buf []byte) (int, error) { func (r *TraceReader) Read(buf []byte) (int, error) {
n, err := r.r.Read(buf) n, err := r.r.Read(buf)
if n > 0 { if n > 0 {
r.log.Trace(r.level, r.prefix+string(buf[:n])) r.log.Trace(r.level, r.prefix, buf[:n])
} }
return n, err return n, err
} }
func (r *TraceReader) SetTrace(level mlog.Level) { func (r *TraceReader) SetTrace(level slog.Level) {
r.level = level r.level = level
} }

28
mtasts/mtasts.go
View file

@ -20,6 +20,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -31,8 +33,6 @@ import (
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
) )
var xlog = mlog.New("mtasts")
var ( var (
metricGet = promauto.NewHistogramVec( metricGet = promauto.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
@ -190,11 +190,11 @@ var (
// LookupRecord looks up the MTA-STS TXT DNS record at "_mta-sts.<domain>", // LookupRecord looks up the MTA-STS TXT DNS record at "_mta-sts.<domain>",
// following CNAME records, and returns the parsed MTA-STS record and the DNS TXT // following CNAME records, and returns the parsed MTA-STS record and the DNS TXT
// record. // record.
func LookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) { func LookupRecord(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("mtasts", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("mtasts lookup result", rerr, mlog.Field("domain", domain), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) log.Debugx("mtasts lookup result", rerr, slog.Any("domain", domain), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start)))
}() }()
// ../rfc/8461:289 // ../rfc/8461:289
@ -261,11 +261,11 @@ var HTTPClient = &http.Client{
// //
// If an error is returned, callers should back off for 5 minutes until the next // If an error is returned, callers should back off for 5 minutes until the next
// attempt. // attempt.
func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policyText string, rerr error) { func FetchPolicy(ctx context.Context, elog *slog.Logger, domain dns.Domain) (policy *Policy, policyText string, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("mtasts", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("mtasts fetch policy result", rerr, mlog.Field("domain", domain), mlog.Field("policy", policy), mlog.Field("policytext", policyText), mlog.Field("duration", time.Since(start))) log.Debugx("mtasts fetch policy result", rerr, slog.Any("domain", domain), slog.Any("policy", policy), slog.String("policytext", policyText), slog.Duration("duration", time.Since(start)))
}() }()
// Timeout of 1 minute. ../rfc/8461:569 // Timeout of 1 minute. ../rfc/8461:569
@ -291,7 +291,7 @@ func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policy
// We pass along underlying TLS certificate errors. // We pass along underlying TLS certificate errors.
return nil, "", fmt.Errorf("%w: http get: %w", ErrPolicyFetch, err) return nil, "", fmt.Errorf("%w: http get: %w", ErrPolicyFetch, err)
} }
metrics.HTTPClientObserve(ctx, "mtasts", req.Method, resp.StatusCode, err, start) metrics.HTTPClientObserve(ctx, log, "mtasts", req.Method, resp.StatusCode, err, start)
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound { if resp.StatusCode == http.StatusNotFound {
return nil, "", ErrNoPolicy return nil, "", ErrNoPolicy
@ -329,22 +329,22 @@ func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policy
// record is still returned. // record is still returned.
// //
// Also see Get in package mtastsdb. // Also see Get in package mtastsdb.
func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (record *Record, policy *Policy, policyText string, err error) { func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (record *Record, policy *Policy, policyText string, err error) {
log := xlog.WithContext(ctx) log := mlog.New("mtasts", elog)
start := time.Now() start := time.Now()
result := "lookuperror" result := "lookuperror"
defer func() { defer func() {
metricGet.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) metricGet.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("mtasts get result", err, mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("policy", policy), mlog.Field("duration", time.Since(start))) log.Debugx("mtasts get result", err, slog.Any("domain", domain), slog.Any("record", record), slog.Any("policy", policy), slog.Duration("duration", time.Since(start)))
}() }()
record, _, err = LookupRecord(ctx, resolver, domain) record, _, err = LookupRecord(ctx, log.Logger, resolver, domain)
if err != nil { if err != nil {
return nil, nil, "", err return nil, nil, "", err
} }
result = "fetcherror" result = "fetcherror"
policy, policyText, err = FetchPolicy(ctx, domain) policy, policyText, err = FetchPolicy(ctx, log.Logger, domain)
if err != nil { if err != nil {
return record, nil, "", err return record, nil, "", err
} }

17
mtasts/mtasts_test.go
View file

@ -8,7 +8,7 @@ import (
"crypto/x509" "crypto/x509"
"errors" "errors"
"io" "io"
"log" golog "log"
"math/big" "math/big"
"net" "net"
"net/http" "net/http"
@ -18,6 +18,8 @@ import (
"testing" "testing"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
@ -25,7 +27,8 @@ import (
) )
func TestLookup(t *testing.T) { func TestLookup(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug})
log := mlog.New("mtasts", nil)
resolver := dns.MockResolver{ resolver := dns.MockResolver{
TXT: map[string][]string{ TXT: map[string][]string{
@ -50,7 +53,7 @@ func TestLookup(t *testing.T) {
test := func(host string, expRecord *Record, expErr error) { test := func(host string, expRecord *Record, expErr error) {
t.Helper() t.Helper()
record, _, err := LookupRecord(context.Background(), resolver, dns.Domain{ASCII: host}) record, _, err := LookupRecord(context.Background(), log.Logger, resolver, dns.Domain{ASCII: host})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("lookup: got err %#v, expected %#v", err, expErr) t.Fatalf("lookup: got err %#v, expected %#v", err, expErr)
} }
@ -184,6 +187,8 @@ func fakeCert(t *testing.T, expired bool) tls.Certificate {
} }
func TestFetch(t *testing.T) { func TestFetch(t *testing.T) {
log := mlog.New("mtasts", nil)
certok := fakeCert(t, false) certok := fakeCert(t, false)
certbad := fakeCert(t, true) certbad := fakeCert(t, true)
@ -218,7 +223,7 @@ func TestFetch(t *testing.T) {
TLSConfig: &tls.Config{ TLSConfig: &tls.Config{
Certificates: []tls.Certificate{cert}, Certificates: []tls.Certificate{cert},
}, },
ErrorLog: log.New(io.Discard, "", 0), ErrorLog: golog.New(io.Discard, "", 0),
} }
s.ServeTLS(l, "", "") s.ServeTLS(l, "", "")
}() }()
@ -235,7 +240,7 @@ func TestFetch(t *testing.T) {
}, },
} }
p, _, err := FetchPolicy(context.Background(), dns.Domain{ASCII: domain}) p, _, err := FetchPolicy(context.Background(), log.Logger, dns.Domain{ASCII: domain})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("policy: got err %#v, expected %#v", err, expErr) t.Fatalf("policy: got err %#v, expected %#v", err, expErr)
} }
@ -247,7 +252,7 @@ func TestFetch(t *testing.T) {
expErr = ErrNoRecord expErr = ErrNoRecord
} }
_, p, _, err = Get(context.Background(), resolver, dns.Domain{ASCII: domain}) _, p, _, err = Get(context.Background(), log.Logger, resolver, dns.Domain{ASCII: domain})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("get: got err %#v, expected %#v", err, expErr) t.Fatalf("get: got err %#v, expected %#v", err, expErr)
} }

23
mtastsdb/db.go
View file

@ -16,6 +16,8 @@ import (
"sync" "sync"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -28,8 +30,6 @@ import (
"github.com/mjl-/mox/tlsrpt" "github.com/mjl-/mox/tlsrpt"
) )
var xlog = mlog.New("mtastsdb")
var ( var (
metricGet = promauto.NewCounterVec( metricGet = promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
@ -108,7 +108,7 @@ func Close() {
defer mutex.Unlock() defer mutex.Unlock()
if DB != nil { if DB != nil {
err := DB.Close() err := DB.Close()
xlog.Check(err, "closing database") mlog.New("mtastsdb", nil).Check(err, "closing database")
DB = nil DB = nil
} }
} }
@ -119,8 +119,7 @@ func Close() {
// //
// Returns ErrNotFound if record is not present. // Returns ErrNotFound if record is not present.
// Returns ErrBackoff if a recent attempt to fetch a record failed. // Returns ErrBackoff if a recent attempt to fetch a record failed.
func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) { func lookup(ctx context.Context, log mlog.Log, domain dns.Domain) (*PolicyRecord, error) {
log := xlog.WithContext(ctx)
db, err := database(ctx) db, err := database(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
@ -222,8 +221,8 @@ func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) {
// Get returns an "sts" or "no-policy-found" in reportResult in most cases (when // Get returns an "sts" or "no-policy-found" in reportResult in most cases (when
// not a local/internal error). It may add an "sts" result without policy contents // not a local/internal error). It may add an "sts" result without policy contents
// ("policy-string") in case of errors while fetching the policy. // ("policy-string") in case of errors while fetching the policy.
func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) { func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) {
log := xlog.WithContext(ctx) log := mlog.New("mtastsdb", elog)
defer func() { defer func() {
result := "ok" result := "ok"
if err != nil && errors.Is(err, ErrBackoff) { if err != nil && errors.Is(err, ErrBackoff) {
@ -234,16 +233,16 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
result = "error" result = "error"
} }
metricGet.WithLabelValues(result).Inc() metricGet.WithLabelValues(result).Inc()
log.Debugx("mtastsdb get result", err, mlog.Field("domain", domain), mlog.Field("fresh", fresh)) log.Debugx("mtastsdb get result", err, slog.Any("domain", domain), slog.Bool("fresh", fresh))
}() }()
cachedPolicy, err := lookup(ctx, domain) cachedPolicy, err := lookup(ctx, log, domain)
if err != nil && errors.Is(err, ErrNotFound) { if err != nil && errors.Is(err, ErrNotFound) {
// We don't have a policy for this domain, not even a record that we tried recently // We don't have a policy for this domain, not even a record that we tried recently
// and should backoff. So attempt to fetch policy. // and should backoff. So attempt to fetch policy.
nctx, cancel := context.WithTimeout(ctx, time.Minute) nctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel() defer cancel()
record, p, ptext, err := mtasts.Get(nctx, resolver, domain) record, p, ptext, err := mtasts.Get(nctx, log.Logger, resolver, domain)
if err != nil { if err != nil {
switch { switch {
case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax): case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax):
@ -303,7 +302,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
policy = &cachedPolicy.Policy policy = &cachedPolicy.Policy
nctx, cancel := context.WithTimeout(ctx, 30*time.Second) nctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel() defer cancel()
record, _, err := mtasts.LookupRecord(nctx, resolver, domain) record, _, err := mtasts.LookupRecord(nctx, log.Logger, resolver, domain)
if err != nil { if err != nil {
if errors.Is(err, mtasts.ErrNoRecord) { if errors.Is(err, mtasts.ErrNoRecord) {
if policy.Mode != mtasts.ModeNone { if policy.Mode != mtasts.ModeNone {
@ -336,7 +335,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
// didn't store the raw policy lines in the past. // didn't store the raw policy lines in the past.
nctx, cancel = context.WithTimeout(ctx, 30*time.Second) nctx, cancel = context.WithTimeout(ctx, 30*time.Second)
defer cancel() defer cancel()
p, ptext, err := mtasts.FetchPolicy(nctx, domain) p, ptext, err := mtasts.FetchPolicy(nctx, log.Logger, domain)
if err != nil { if err != nil {
log.Errorx("fetching updated policy for domain, continuing with previously cached policy", err) log.Errorx("fetching updated policy for domain, continuing with previously cached policy", err)

13
mtastsdb/db_test.go
View file

@ -12,6 +12,7 @@ import (
"time" "time"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/mtasts" "github.com/mjl-/mox/mtasts"
) )
@ -32,6 +33,8 @@ func TestDB(t *testing.T) {
os.Remove(dbpath) os.Remove(dbpath)
defer os.Remove(dbpath) defer os.Remove(dbpath)
log := mlog.New("mtastsdb", nil)
if err := Init(false); err != nil { if err := Init(false); err != nil {
t.Fatalf("init database: %s", err) t.Fatalf("init database: %s", err)
} }
@ -42,7 +45,7 @@ func TestDB(t *testing.T) {
timeNow = func() time.Time { return now } timeNow = func() time.Time { return now }
defer func() { timeNow = time.Now }() defer func() { timeNow = time.Now }()
if p, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != ErrNotFound { if p, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != ErrNotFound {
t.Fatalf("expected not found, got %v, %#v", err, p) t.Fatalf("expected not found, got %v, %#v", err, p)
} }
@ -59,7 +62,7 @@ func TestDB(t *testing.T) {
if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "123", &policy1, policy1.String()); err != nil { if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "123", &policy1, policy1.String()); err != nil {
t.Fatalf("upsert record: %s", err) t.Fatalf("upsert record: %s", err)
} }
if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil { if got, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != nil {
t.Fatalf("lookup after insert: %s", err) t.Fatalf("lookup after insert: %s", err)
} else if !reflect.DeepEqual(got.Policy, policy1) { } else if !reflect.DeepEqual(got.Policy, policy1) {
t.Fatalf("mismatch between inserted and retrieved: got %#v, want %#v", got, policy1) t.Fatalf("mismatch between inserted and retrieved: got %#v, want %#v", got, policy1)
@ -76,7 +79,7 @@ func TestDB(t *testing.T) {
if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "124", &policy2, policy2.String()); err != nil { if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "124", &policy2, policy2.String()); err != nil {
t.Fatalf("upsert record: %s", err) t.Fatalf("upsert record: %s", err)
} }
if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil { if got, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != nil {
t.Fatalf("lookup after insert: %s", err) t.Fatalf("lookup after insert: %s", err)
} else if !reflect.DeepEqual(got.Policy, policy2) { } else if !reflect.DeepEqual(got.Policy, policy2) {
t.Fatalf("mismatch between inserted and retrieved: got %v, want %v", got, policy2) t.Fatalf("mismatch between inserted and retrieved: got %v, want %v", got, policy2)
@ -108,7 +111,7 @@ func TestDB(t *testing.T) {
t.Fatalf("records mismatch, got %#v, expected %#v", records, expRecords) t.Fatalf("records mismatch, got %#v, expected %#v", records, expRecords)
} }
if _, err := lookup(ctxbg, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff { if _, err := lookup(ctxbg, log, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff {
t.Fatalf("got %#v, expected ErrBackoff", err) t.Fatalf("got %#v, expected ErrBackoff", err)
} }
@ -125,7 +128,7 @@ func TestDB(t *testing.T) {
testGet := func(domain string, expPolicy *mtasts.Policy, expFresh bool, expErr error) { testGet := func(domain string, expPolicy *mtasts.Policy, expFresh bool, expErr error) {
t.Helper() t.Helper()
p, _, fresh, err := Get(ctxbg, resolver, dns.Domain{ASCII: domain}) p, _, fresh, err := Get(ctxbg, log.Logger, resolver, dns.Domain{ASCII: domain})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %v, expected %v", err, expErr) t.Fatalf("got err %v, expected %v", err, expErr)
} }

37
mtastsdb/refresh.go
View file

@ -8,6 +8,8 @@ import (
"runtime/debug" "runtime/debug"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
@ -28,11 +30,9 @@ func refresh() int {
for { for {
ticker.Reset(interval) ticker.Reset(interval)
ctx := context.WithValue(mox.Context, mlog.CidKey, mox.Cid()) log := mlog.New("mtastsdb", nil).WithCid(mox.Cid())
n, err := refresh1(ctx, dns.StrictResolver{Pkg: "mtastsdb"}, time.Sleep) n, err := refresh1(mox.Context, log, dns.StrictResolver{Pkg: "mtastsdb"}, time.Sleep)
if err != nil { log.Check(err, "periodic refresh of cached mtasts policies")
xlog.WithContext(ctx).Errorx("periodic refresh of cached mtasts policies", err)
}
if n > 0 { if n > 0 {
refreshed += n refreshed += n
} }
@ -51,7 +51,7 @@ func refresh() int {
// refreshes evenly over the next 3 hours, randomizing the domains, and we add some // refreshes evenly over the next 3 hours, randomizing the domains, and we add some
// jitter to the timing. Each refresh is done in a new goroutine, so a single slow // jitter to the timing. Each refresh is done in a new goroutine, so a single slow
// refresh doesn't mess up the timing. // refresh doesn't mess up the timing.
func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) { func refresh1(ctx context.Context, log mlog.Log, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) {
db, err := database(ctx) db, err := database(ctx)
if err != nil { if err != nil {
return 0, err return 0, err
@ -87,10 +87,10 @@ func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Dura
} }
// Launch goroutine with the refresh. // Launch goroutine with the refresh.
xlog.WithContext(ctx).Debug("will refresh mta-sts policies over next 3 hours", mlog.Field("count", len(prs))) log.Debug("will refresh mta-sts policies over next 3 hours", slog.Int("count", len(prs)))
start := timeNow() start := timeNow()
for i, pr := range prs { for i, pr := range prs {
go refreshDomain(ctx, db, resolver, pr) go refreshDomain(ctx, log, db, resolver, pr)
if i < len(prs)-1 { if i < len(prs)-1 {
interval := 3 * int64(time.Hour) / int64(len(prs)-1) interval := 3 * int64(time.Hour) / int64(len(prs)-1)
extra := time.Duration(rand.Int63n(interval) - interval/2) extra := time.Duration(rand.Int63n(interval) - interval/2)
@ -104,13 +104,12 @@ func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Dura
return len(prs), nil return len(prs), nil
} }
func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr PolicyRecord) { func refreshDomain(ctx context.Context, log mlog.Log, db *bstore.DB, resolver dns.Resolver, pr PolicyRecord) {
log := xlog.WithContext(ctx)
defer func() { defer func() {
x := recover() x := recover()
if x != nil { if x != nil {
// Should not happen, but make sure errors don't take down the application. // Should not happen, but make sure errors don't take down the application.
log.Error("refresh1", mlog.Field("panic", x)) log.Error("refresh1", slog.Any("panic", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Mtastsdb) metrics.PanicInc(metrics.Mtastsdb)
} }
@ -121,11 +120,11 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
d, err := dns.ParseDomain(pr.Domain) d, err := dns.ParseDomain(pr.Domain)
if err != nil { if err != nil {
log.Errorx("refreshing mta-sts policy: parsing policy domain", err, mlog.Field("domain", d)) log.Errorx("refreshing mta-sts policy: parsing policy domain", err, slog.Any("domain", d))
return return
} }
log.Debug("refreshing mta-sts policy for domain", mlog.Field("domain", d)) log.Debug("refreshing mta-sts policy for domain", slog.Any("domain", d))
record, _, err := mtasts.LookupRecord(ctx, resolver, d) record, _, err := mtasts.LookupRecord(ctx, log.Logger, resolver, d)
if err == nil && record.ID == pr.RecordID { if err == nil && record.ID == pr.RecordID {
qup := bstore.QueryDB[PolicyRecord](ctx, db) qup := bstore.QueryDB[PolicyRecord](ctx, db)
qup.FilterNonzero(PolicyRecord{Domain: pr.Domain, LastUpdate: pr.LastUpdate}) qup.FilterNonzero(PolicyRecord{Domain: pr.Domain, LastUpdate: pr.LastUpdate})
@ -137,7 +136,7 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
if n, err := qup.UpdateNonzero(update); err != nil { if n, err := qup.UpdateNonzero(update); err != nil {
log.Errorx("updating refreshed, unmodified policy in database", err) log.Errorx("updating refreshed, unmodified policy in database", err)
} else if n != 1 { } else if n != 1 {
log.Info("expected to update 1 policy after refresh", mlog.Field("count", n)) log.Info("expected to update 1 policy after refresh", slog.Int("count", n))
} }
return return
} }
@ -152,14 +151,14 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
// ../rfc/8461:587 // ../rfc/8461:587
return return
} else if err != nil { } else if err != nil {
log.Errorx("looking up mta-sts record for domain", err, mlog.Field("domain", d)) log.Errorx("looking up mta-sts record for domain", err, slog.Any("domain", d))
// Try to fetch new policy. It could be just DNS that is down. We don't want to let our policy expire. // Try to fetch new policy. It could be just DNS that is down. We don't want to let our policy expire.
} }
p, _, err := mtasts.FetchPolicy(ctx, d) p, _, err := mtasts.FetchPolicy(ctx, log.Logger, d)
if err != nil { if err != nil {
if !errors.Is(err, mtasts.ErrNoPolicy) || pr.Mode != mtasts.ModeNone { if !errors.Is(err, mtasts.ErrNoPolicy) || pr.Mode != mtasts.ModeNone {
log.Errorx("refreshing mtasts policy for domain", err, mlog.Field("domain", d)) log.Errorx("refreshing mtasts policy for domain", err, slog.Any("domain", d))
} }
return return
} }
@ -178,6 +177,6 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
if n, err := qup.UpdateFields(update); err != nil { if n, err := qup.UpdateFields(update); err != nil {
log.Errorx("updating refreshed, modified policy in database", err) log.Errorx("updating refreshed, modified policy in database", err)
} else if n != 1 { } else if n != 1 {
log.Info("updating refreshed, did not update 1 policy", mlog.Field("count", n)) log.Info("updating refreshed, did not update 1 policy", slog.Int("count", n))
} }
} }

4
mtastsdb/refresh_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/mtasts" "github.com/mjl-/mox/mtasts"
) )
@ -135,7 +136,8 @@ func TestRefresh(t *testing.T) {
t.Fatalf("bad sleep duration %v", d) t.Fatalf("bad sleep duration %v", d)
} }
} }
if n, err := refresh1(ctxbg, resolver, sleep); err != nil || n != 3 { log := mlog.New("mtastsdb", nil)
if n, err := refresh1(ctxbg, log, resolver, sleep); err != nil || n != 3 {
t.Fatalf("refresh1: err %s, n %d, expected no error, 3", err, n) t.Fatalf("refresh1: err %s, n %d, expected no error, 3", err, n)
} }
if slept != 2 { if slept != 2 {

30
publicsuffix/list.go
View file

@ -18,14 +18,13 @@ import (
_ "embed" _ "embed"
"golang.org/x/exp/slog"
"golang.org/x/net/idna" "golang.org/x/net/idna"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
var xlog = mlog.New("publicsuffix")
// todo: automatically fetch new lists periodically? compare it with the old one. refuse it if it changed too much, especially if it contains far fewer entries than before. // todo: automatically fetch new lists periodically? compare it with the old one. refuse it if it changed too much, especially if it contains far fewer entries than before.
// Labels map from utf8 labels to labels for subdomains. // Labels map from utf8 labels to labels for subdomains.
@ -43,16 +42,19 @@ var publicsuffixList List
var publicsuffixData []byte var publicsuffixData []byte
func init() { func init() {
l, err := ParseList(bytes.NewReader(publicsuffixData)) log := mlog.New("publicsuffix", nil)
l, err := ParseList(log.Logger, bytes.NewReader(publicsuffixData))
if err != nil { if err != nil {
xlog.Fatalx("parsing public suffix list", err) log.Fatalx("parsing public suffix list", err)
} }
publicsuffixList = l publicsuffixList = l
} }
// ParseList parses a public suffix list. // ParseList parses a public suffix list.
// Only the "ICANN DOMAINS" are used. // Only the "ICANN DOMAINS" are used.
func ParseList(r io.Reader) (List, error) { func ParseList(elog *slog.Logger, r io.Reader) (List, error) {
log := mlog.New("publicsuffix", elog)
list := List{labels{}, labels{}} list := List{labels{}, labels{}}
br := bufio.NewReader(r) br := bufio.NewReader(r)
@ -79,7 +81,7 @@ func ParseList(r io.Reader) (List, error) {
l = list.excludes l = list.excludes
t = strings.Split(line, ".") t = strings.Split(line, ".")
if len(t) == 1 { if len(t) == 1 {
xlog.Print("exclude rule with single label, skipping", mlog.Field("line", oline)) log.Print("exclude rule with single label, skipping", slog.String("line", oline))
continue continue
} }
} else { } else {
@ -88,19 +90,19 @@ func ParseList(r io.Reader) (List, error) {
for i := len(t) - 1; i >= 0; i-- { for i := len(t) - 1; i >= 0; i-- {
w := t[i] w := t[i]
if w == "" { if w == "" {
xlog.Print("empty label in rule, skipping", mlog.Field("line", oline)) log.Print("empty label in rule, skipping", slog.String("line", oline))
break break
} }
if w != "" && w != "*" { if w != "" && w != "*" {
w, err = idna.Lookup.ToUnicode(w) w, err = idna.Lookup.ToUnicode(w)
if err != nil { if err != nil {
xlog.Printx("invalid label, skipping", err, mlog.Field("line", oline)) log.Printx("invalid label, skipping", err, slog.String("line", oline))
} }
} }
m, ok := l[w] m, ok := l[w]
if ok { if ok {
if _, dup := m[""]; i == 0 && dup { if _, dup := m[""]; i == 0 && dup {
xlog.Print("duplicate rule", mlog.Field("line", oline)) log.Print("duplicate rule", slog.String("line", oline))
} }
l = m l = m
} else { } else {
@ -123,16 +125,16 @@ func ParseList(r io.Reader) (List, error) {
// Lookup calls Lookup on the builtin public suffix list, from // Lookup calls Lookup on the builtin public suffix list, from
// https://publicsuffix.org/list/. // https://publicsuffix.org/list/.
func Lookup(ctx context.Context, domain dns.Domain) (orgDomain dns.Domain) { func Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) {
return publicsuffixList.Lookup(ctx, domain) return publicsuffixList.Lookup(ctx, elog, domain)
} }
// Lookup returns the organizational domain. If domain is an organizational // Lookup returns the organizational domain. If domain is an organizational
// domain, or higher-level, the same domain is returned. // domain, or higher-level, the same domain is returned.
func (l List) Lookup(ctx context.Context, domain dns.Domain) (orgDomain dns.Domain) { func (l List) Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) {
log := xlog.WithContext(ctx) log := mlog.New("publicsuffix", elog)
defer func() { defer func() {
log.Debug("publicsuffix lookup result", mlog.Field("reqdom", domain), mlog.Field("orgdom", orgDomain)) log.Debug("publicsuffix lookup result", slog.Any("reqdom", domain), slog.Any("orgdom", orgDomain))
}() }()
t := strings.Split(domain.Name(), ".") t := strings.Split(domain.Name(), ".")

10
publicsuffix/list_test.go
View file

@ -7,6 +7,7 @@ import (
"testing" "testing"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
) )
func TestList(t *testing.T) { func TestList(t *testing.T) {
@ -27,7 +28,10 @@ bücher.example.com
ignored.example.com ignored.example.com
` `
l, err := ParseList(strings.NewReader(data))
log := mlog.New("publicsuffix", nil)
l, err := ParseList(log.Logger, strings.NewReader(data))
if err != nil { if err != nil {
t.Fatalf("parsing list: %s", err) t.Fatalf("parsing list: %s", err)
} }
@ -44,7 +48,7 @@ ignored.example.com
t.Fatalf("idna to unicode org domain %q: %s", orgDomain, err) t.Fatalf("idna to unicode org domain %q: %s", orgDomain, err)
} }
r := l.Lookup(context.Background(), d) r := l.Lookup(context.Background(), log.Logger, d)
if r != od { if r != od {
t.Fatalf("got %q, expected %q, for domain %q", r, orgDomain, domain) t.Fatalf("got %q, expected %q, for domain %q", r, orgDomain, domain)
} }
@ -70,7 +74,7 @@ ignored.example.com
test("bar.foo.xn--bcher-kva.example.com", "foo.bücher.example.com") test("bar.foo.xn--bcher-kva.example.com", "foo.bücher.example.com")
test("x.ignored.example.com", "example.com") test("x.ignored.example.com", "example.com")
l, err = ParseList(bytes.NewReader(publicsuffixData)) l, err = ParseList(log.Logger, bytes.NewReader(publicsuffixData))
if err != nil { if err != nil {
t.Fatalf("parsing public suffix list: %s", err) t.Fatalf("parsing public suffix list: %s", err)
} }

93
queue/direct.go
View file

@ -11,6 +11,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -88,7 +90,7 @@ var (
) )
// todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time? // todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time?
func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { func fail(qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
// todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503 // todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503
// todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls? // todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls?
// todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379 // todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379
@ -106,18 +108,18 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
qup := bstore.QueryDB[Msg](context.Background(), DB) qup := bstore.QueryDB[Msg](context.Background(), DB)
qup.FilterID(m.ID) qup.FilterID(m.ID)
if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil { if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil {
qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg)) qlog.Errorx("storing delivery error", err, slog.String("deliveryerror", errmsg))
} }
if m.Attempts == 5 { if m.Attempts == 5 {
// We've attempted deliveries at these intervals: 0, 7.5m, 15m, 30m, 1h, 2u. // We've attempted deliveries at these intervals: 0, 7.5m, 15m, 30m, 1h, 2u.
// Let sender know delivery is delayed. // Let sender know delivery is delayed.
qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), mlog.Field("backoff", backoff)) qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), slog.Duration("backoff", backoff))
retryUntil := m.LastAttempt.Add((4 + 8 + 16) * time.Hour) retryUntil := m.LastAttempt.Add((4 + 8 + 16) * time.Hour)
deliverDSNDelay(qlog, m, remoteMTA, secodeOpt, errmsg, retryUntil) deliverDSNDelay(qlog, m, remoteMTA, secodeOpt, errmsg, retryUntil)
} else { } else {
qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), mlog.Field("backoff", backoff), mlog.Field("nextattempt", m.NextAttempt)) qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), slog.Duration("backoff", backoff), slog.Time("nextattempt", m.NextAttempt))
} }
} }
@ -129,7 +131,7 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
// domain (MTA-STS), its policy type can be empty, in which case there is no // domain (MTA-STS), its policy type can be empty, in which case there is no
// information (e.g. internal failure). hostResults are per-host details (DANE, one // information (e.g. internal failure). hostResults are per-host details (DANE, one
// per MX target). // per MX target).
func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) (recipientDomainResult tlsrpt.Result, hostResults []tlsrpt.Result) { func deliverDirect(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) (recipientDomainResult tlsrpt.Result, hostResults []tlsrpt.Result) {
// High-level approach: // High-level approach:
// - Resolve domain to deliver to (CNAME), and determine hosts to try to deliver to (MX) // - Resolve domain to deliver to (CNAME), and determine hosts to try to deliver to (MX)
// - Get MTA-STS policy for domain (optional). If present, only deliver to its // - Get MTA-STS policy for domain (optional). If present, only deliver to its
@ -151,8 +153,8 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
// possibly a chain. If there are no MX records, it can be an IP or the host // possibly a chain. If there are no MX records, it can be an IP or the host
// directly. // directly.
origNextHop := m.RecipientDomain.Domain origNextHop := m.RecipientDomain.Domain
ctx := context.WithValue(mox.Context, mlog.CidKey, cid) ctx := mox.Shutdown
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog, resolver, m.RecipientDomain) haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog.Logger, resolver, m.RecipientDomain)
if err != nil { if err != nil {
// If this is a DNSSEC authentication error, we'll collect it for TLS reporting. // If this is a DNSSEC authentication error, we'll collect it for TLS reporting.
// Hopefully it's a temporary misconfiguration that is solve before we try to send // Hopefully it's a temporary misconfiguration that is solve before we try to send
@ -179,14 +181,13 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
// would only take a single CNAME DNS response to direct us to an unrelated domain. // would only take a single CNAME DNS response to direct us to an unrelated domain.
var policy *mtasts.Policy // Policy can have mode enforce, testing and none. var policy *mtasts.Policy // Policy can have mode enforce, testing and none.
if !origNextHop.IsZero() { if !origNextHop.IsZero() {
cidctx := context.WithValue(mox.Shutdown, mlog.CidKey, cid) policy, recipientDomainResult, _, err = mtastsdb.Get(ctx, qlog.Logger, resolver, origNextHop)
policy, recipientDomainResult, _, err = mtastsdb.Get(cidctx, resolver, origNextHop)
if err != nil { if err != nil {
if tlsRequiredNo { if tlsRequiredNo {
qlog.Infox("mtasts lookup temporary error, continuing due to tls-required-no message header", err, mlog.Field("domain", origNextHop)) qlog.Infox("mtasts lookup temporary error, continuing due to tls-required-no message header", err, slog.Any("domain", origNextHop))
metricTLSRequiredNoIgnored.WithLabelValues("mtastspolicy").Inc() metricTLSRequiredNoIgnored.WithLabelValues("mtastspolicy").Inc()
} else { } else {
qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, mlog.Field("domain", origNextHop)) qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, slog.Any("domain", origNextHop))
recipientDomainResult.Summary.TotalFailureSessionCount++ recipientDomainResult.Summary.TotalFailureSessionCount++
fail(qlog, m, backoff, false, dsn.NameIP{}, "", err.Error()) fail(qlog, m, backoff, false, dsn.NameIP{}, "", err.Error())
return return
@ -226,22 +227,21 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
} }
if policy.Mode == mtasts.ModeEnforce { if policy.Mode == mtasts.ModeEnforce {
if tlsRequiredNo { if tlsRequiredNo {
qlog.Info("mx host does not match mta-sts policy in mode enforce, ignoring due to tls-required-no message header", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts)) qlog.Info("mx host does not match mta-sts policy in mode enforce, ignoring due to tls-required-no message header", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts))
metricTLSRequiredNoIgnored.WithLabelValues("mtastsmx").Inc() metricTLSRequiredNoIgnored.WithLabelValues("mtastsmx").Inc()
} else { } else {
errmsg = fmt.Sprintf("mx host %s does not match enforced mta-sts policy with hosts %s", h.Domain, strings.Join(policyHosts, ",")) errmsg = fmt.Sprintf("mx host %s does not match enforced mta-sts policy with hosts %s", h.Domain, strings.Join(policyHosts, ","))
qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts)) qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts))
recipientDomainResult.Summary.TotalFailureSessionCount++ recipientDomainResult.Summary.TotalFailureSessionCount++
continue continue
} }
} else { } else {
qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts)) qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts))
} }
} }
qlog.Info("delivering to remote", mlog.Field("remote", h), mlog.Field("queuecid", cid)) qlog.Info("delivering to remote", slog.Any("remote", h))
cid := mox.Cid() nqlog := qlog.WithCid(mox.Cid())
nqlog := qlog.WithCid(cid)
var remoteIP net.IP var remoteIP net.IP
enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce
@ -270,7 +270,7 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
var badTLS, ok bool var badTLS, ok bool
var hostResult tlsrpt.Result var hostResult tlsrpt.Result
permanent, tlsDANE, badTLS, secodeOpt, remoteIP, errmsg, hostResult, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode, tlsPKIX, &recipientDomainResult) permanent, tlsDANE, badTLS, secodeOpt, remoteIP, errmsg, hostResult, ok = deliverHost(nqlog, resolver, dialer, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode, tlsPKIX, &recipientDomainResult)
var zerotype tlsrpt.PolicyType var zerotype tlsrpt.PolicyType
if hostResult.Policy.Type != zerotype { if hostResult.Policy.Type != zerotype {
@ -293,8 +293,8 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
} }
// todo future: add a configuration option to not fall back? // todo future: add a configuration option to not fall back?
nqlog.Info("connecting again for delivery attempt without tls", mlog.Field("enforcemtasts", enforceMTASTS), mlog.Field("tlsdane", tlsDANE), mlog.Field("requiretls", m.RequireTLS)) nqlog.Info("connecting again for delivery attempt without tls", slog.Bool("enforcemtasts", enforceMTASTS), slog.Bool("tlsdane", tlsDANE), slog.Any("requiretls", m.RequireTLS))
permanent, _, _, secodeOpt, remoteIP, errmsg, _, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, smtpclient.TLSSkip, false, &tlsrpt.Result{}) permanent, _, _, secodeOpt, remoteIP, errmsg, _, ok = deliverHost(nqlog, resolver, dialer, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, smtpclient.TLSSkip, false, &tlsrpt.Result{})
} }
if ok { if ok {
@ -350,7 +350,7 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
// The returned hostResult holds TLSRPT reporting results for the connection // The returned hostResult holds TLSRPT reporting results for the connection
// attempt. Its policy type can be the zero value, indicating there was no finding // attempt. Its policy type can be the zero value, indicating there was no finding
// (e.g. internal error). // (e.g. internal error).
func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, cid int64, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode, tlsPKIX bool, recipientDomainResult *tlsrpt.Result) (permanent, tlsDANE, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, hostResult tlsrpt.Result, ok bool) { func deliverHost(log mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode, tlsPKIX bool, recipientDomainResult *tlsrpt.Result) (permanent, tlsDANE, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, hostResult tlsrpt.Result, ok bool) {
// About attempting delivery to multiple addresses of a host: ../rfc/5321:3898 // About attempting delivery to multiple addresses of a host: ../rfc/5321:3898
tlsRequiredNo := m.RequireTLS != nil && !*m.RequireTLS tlsRequiredNo := m.RequireTLS != nil && !*m.RequireTLS
@ -367,18 +367,18 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
} }
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, mode, deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second)) metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, mode, deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debug("queue deliverhost result", log.Debug("queue deliverhost result",
mlog.Field("host", host), slog.Any("host", host),
mlog.Field("attempt", m.Attempts), slog.Int("attempt", m.Attempts),
mlog.Field("tlsmode", tlsMode), slog.Any("tlsmode", tlsMode),
mlog.Field("tlspkix", tlsPKIX), slog.Bool("tlspkix", tlsPKIX),
mlog.Field("tlsdane", tlsDANE), slog.Bool("tlsdane", tlsDANE),
mlog.Field("tlsrequiredno", tlsRequiredNo), slog.Bool("tlsrequiredno", tlsRequiredNo),
mlog.Field("permanent", permanent), slog.Bool("permanent", permanent),
mlog.Field("badtls", badTLS), slog.Bool("badtls", badTLS),
mlog.Field("secodeopt", secodeOpt), slog.String("secodeopt", secodeOpt),
mlog.Field("errmsg", errmsg), slog.String("errmsg", errmsg),
mlog.Field("ok", ok), slog.Bool("ok", ok),
mlog.Field("duration", time.Since(start))) slog.Duration("duration", time.Since(start)))
}() }()
// Open message to deliver. // Open message to deliver.
@ -392,8 +392,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
log.Check(err, "closing message after delivery attempt") log.Check(err, "closing message after delivery attempt")
}() }()
cidctx := context.WithValue(mox.Context, mlog.CidKey, cid) ctx, cancel := context.WithTimeout(mox.Shutdown, 30*time.Second)
ctx, cancel := context.WithTimeout(cidctx, 30*time.Second)
defer cancel() defer cancel()
// We must lookup the IPs for the host name before checking DANE TLSA records. And // We must lookup the IPs for the host name before checking DANE TLSA records. And
@ -415,10 +414,10 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
} }
metricDestinations.Inc() metricDestinations.Inc()
authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log, resolver, host, m.DialedIPs) authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log.Logger, resolver, host, m.DialedIPs)
destAuthentic := err == nil && authentic && origNextHopAuthentic && (!haveMX || expandedNextHopAuthentic) && host.IsDomain() destAuthentic := err == nil && authentic && origNextHopAuthentic && (!haveMX || expandedNextHopAuthentic) && host.IsDomain()
if !destAuthentic { if !destAuthentic {
log.Debugx("not attempting verification with dane", err, mlog.Field("authentic", authentic), mlog.Field("expandedauthentic", expandedAuthentic)) log.Debugx("not attempting verification with dane", err, slog.Bool("authentic", authentic), slog.Bool("expandedauthentic", expandedAuthentic))
// Track a DNSSEC error if found. // Track a DNSSEC error if found.
var errCode adns.ErrorCode var errCode adns.ErrorCode
@ -447,7 +446,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
// Look for TLSA records in either the expandedHost, or otherwise the original // Look for TLSA records in either the expandedHost, or otherwise the original
// host. ../rfc/7672:912 // host. ../rfc/7672:912
var tlsaBaseDomain dns.Domain var tlsaBaseDomain dns.Domain
tlsDANE, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost) tlsDANE, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log.Logger, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost)
if tlsDANE { if tlsDANE {
metricDestinationDANERequired.Inc() metricDestinationDANERequired.Inc()
} }
@ -475,7 +474,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
}, },
} }
} else { } else {
log.Debug("delivery with required starttls with dane verification", mlog.Field("allowedtlshostnames", tlsHostnames)) log.Debug("delivery with required starttls with dane verification", slog.Any("allowedtlshostnames", tlsHostnames))
} }
// Based on CNAMEs followed and DNSSEC-secure status, we must allow up to 4 host // Based on CNAMEs followed and DNSSEC-secure status, we must allow up to 4 host
// names. // names.
@ -525,7 +524,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
if m.DialedIPs == nil { if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{} m.DialedIPs = map[string][]net.IP{}
} }
conn, remoteIP, err = smtpclient.Dial(ctx, log, dialer, host, ips, 25, m.DialedIPs) conn, remoteIP, err = smtpclient.Dial(ctx, log.Logger, dialer, host, ips, 25, m.DialedIPs)
} }
cancel() cancel()
@ -543,7 +542,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
} }
metricConnection.WithLabelValues(result).Inc() metricConnection.WithLabelValues(result).Inc()
if err != nil { if err != nil {
log.Debugx("connecting to remote smtp", err, mlog.Field("host", host)) log.Debugx("connecting to remote smtp", err, slog.Any("host", host))
return false, tlsDANE, false, "", remoteIP, fmt.Sprintf("dialing smtp server: %v", err), hostResult, false return false, tlsDANE, false, "", remoteIP, fmt.Sprintf("dialing smtp server: %v", err), hostResult, false
} }
@ -554,8 +553,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
rcptTo := m.Recipient().XString(m.SMTPUTF8) rcptTo := m.Recipient().XString(m.SMTPUTF8)
// todo future: get closer to timeouts specified in rfc? ../rfc/5321:3610 // todo future: get closer to timeouts specified in rfc? ../rfc/5321:3610
log = log.Fields(mlog.Field("remoteip", remoteIP)) log = log.With(slog.Any("remoteip", remoteIP))
ctx, cancel = context.WithTimeout(cidctx, 30*time.Minute) ctx, cancel = context.WithTimeout(mox.Shutdown, 30*time.Minute)
defer cancel() defer cancel()
mox.Connections.Register(conn, "smtpclient", "queue") mox.Connections.Register(conn, "smtpclient", "queue")
@ -577,7 +576,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
RecipientDomainResult: recipientDomainResult, RecipientDomainResult: recipientDomainResult,
HostResult: &hostResult, HostResult: &hostResult,
} }
sc, err := smtpclient.New(ctx, log, conn, tlsMode, tlsPKIX, ourHostname, firstHost, opts) sc, err := smtpclient.New(ctx, log.Logger, conn, tlsMode, tlsPKIX, ourHostname, firstHost, opts)
defer func() { defer func() {
if sc == nil { if sc == nil {
conn.Close() conn.Close()
@ -597,7 +596,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
STARTTLS: sc.TLSEnabled(), STARTTLS: sc.TLSEnabled(),
RequireTLS: sc.SupportsRequireTLS(), RequireTLS: sc.SupportsRequireTLS(),
} }
if err = updateRecipientDomainTLS(ctx, m.SenderAccount, rdt); err != nil { if err = updateRecipientDomainTLS(ctx, log, m.SenderAccount, rdt); err != nil {
err = fmt.Errorf("storing recipient domain tls status: %w", err) err = fmt.Errorf("storing recipient domain tls status: %w", err)
} }
} }
@ -658,8 +657,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
} }
// Update (overwite) last known starttls/requiretls support for recipient domain. // Update (overwite) last known starttls/requiretls support for recipient domain.
func updateRecipientDomainTLS(ctx context.Context, senderAccount string, rdt store.RecipientDomainTLS) error { func updateRecipientDomainTLS(ctx context.Context, log mlog.Log, senderAccount string, rdt store.RecipientDomainTLS) error {
acc, err := store.OpenAccount(senderAccount) acc, err := store.OpenAccount(log, senderAccount)
if err != nil { if err != nil {
return fmt.Errorf("open account: %w", err) return fmt.Errorf("open account: %w", err)
} }

18
queue/dsn.go
View file

@ -6,6 +6,8 @@ import (
"os" "os"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -27,7 +29,7 @@ var (
) )
) )
func deliverDSNFailure(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { func deliverDSNFailure(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
const subject = "mail delivery failed" const subject = "mail delivery failed"
message := fmt.Sprintf(` message := fmt.Sprintf(`
Delivery has failed permanently for your email to: Delivery has failed permanently for your email to:
@ -44,7 +46,7 @@ Error during the last delivery attempt:
deliverDSN(log, m, remoteMTA, secodeOpt, errmsg, true, nil, subject, message) deliverDSN(log, m, remoteMTA, secodeOpt, errmsg, true, nil, subject, message)
} }
func deliverDSNDelay(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) { func deliverDSNDelay(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) {
// Should not happen, but doesn't hurt to prevent sending delayed delivery // Should not happen, but doesn't hurt to prevent sending delayed delivery
// notifications for DMARC reports. We don't want to waste postmaster attention. // notifications for DMARC reports. We don't want to waste postmaster attention.
if m.IsDMARCReport { if m.IsDMARCReport {
@ -72,14 +74,14 @@ Error during the last delivery attempt:
// users. So we are delivering to local users. ../rfc/5321:1466 // users. So we are delivering to local users. ../rfc/5321:1466
// ../rfc/5321:1494 // ../rfc/5321:1494
// ../rfc/7208:490 // ../rfc/7208:490
func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) { func deliverDSN(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) {
kind := "delayed delivery" kind := "delayed delivery"
if permanent { if permanent {
kind = "failure" kind = "failure"
} }
qlog := func(text string, err error) { qlog := func(text string, err error) {
log.Errorx("queue dsn: "+text+": sender will not be informed about dsn", err, mlog.Field("sender", m.Sender().XString(m.SMTPUTF8)), mlog.Field("kind", kind)) log.Errorx("queue dsn: "+text+": sender will not be informed about dsn", err, slog.String("sender", m.Sender().XString(m.SMTPUTF8)), slog.String("kind", kind))
} }
msgf, err := os.Open(m.MessagePath()) msgf, err := os.Open(m.MessagePath())
@ -156,9 +158,9 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
// senderAccount should already by postmaster, but doesn't hurt to ensure it. // senderAccount should already by postmaster, but doesn't hurt to ensure it.
senderAccount = mox.Conf.Static.Postmaster.Account senderAccount = mox.Conf.Static.Postmaster.Account
} }
acc, err := store.OpenAccount(senderAccount) acc, err := store.OpenAccount(log, senderAccount)
if err != nil { if err != nil {
acc, err = store.OpenAccount(mox.Conf.Static.Postmaster.Account) acc, err = store.OpenAccount(log, mox.Conf.Static.Postmaster.Account)
if err != nil { if err != nil {
qlog("looking up postmaster account after sender account was not found", err) qlog("looking up postmaster account after sender account was not found", err)
return return
@ -167,10 +169,10 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
} }
defer func() { defer func() {
err := acc.Close() err := acc.Close()
log.Check(err, "queue dsn: closing account", mlog.Field("sender", m.Sender().XString(m.SMTPUTF8)), mlog.Field("kind", kind)) log.Check(err, "queue dsn: closing account", slog.String("sender", m.Sender().XString(m.SMTPUTF8)), slog.String("kind", kind))
}() }()
msgFile, err := store.CreateMessageTemp("queue-dsn") msgFile, err := store.CreateMessageTemp(log, "queue-dsn")
if err != nil { if err != nil {
qlog("creating temporary message file", err) qlog("creating temporary message file", err)
return return

56
queue/queue.go
View file

@ -15,6 +15,7 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"golang.org/x/net/proxy" "golang.org/x/net/proxy"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
@ -36,8 +37,6 @@ import (
"github.com/mjl-/mox/tlsrptdb" "github.com/mjl-/mox/tlsrptdb"
) )
var xlog = mlog.New("queue")
var ( var (
metricConnection = promauto.NewCounterVec( metricConnection = promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
@ -163,7 +162,9 @@ func Init() error {
// Shutdown closes the queue database. The delivery process isn't stopped. For tests only. // Shutdown closes the queue database. The delivery process isn't stopped. For tests only.
func Shutdown() { func Shutdown() {
err := DB.Close() err := DB.Close()
xlog.Check(err, "closing queue db") if err != nil {
mlog.New("queue", nil).Errorx("closing queue db", err)
}
DB = nil DB = nil
} }
@ -221,7 +222,7 @@ func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf
// //
// Add sets derived fields like RecipientDomainStr, and fields related to queueing, // Add sets derived fields like RecipientDomainStr, and fields related to queueing,
// such as Queued, NextAttempt, LastAttempt, LastError. // such as Queued, NextAttempt, LastAttempt, LastError.
func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error { func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error {
// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759 // todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
if qm.ID != 0 { if qm.ID != 0 {
@ -238,7 +239,7 @@ func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
if qm.SenderAccount == "" { if qm.SenderAccount == "" {
return fmt.Errorf("cannot queue with localserve without local account") return fmt.Errorf("cannot queue with localserve without local account")
} }
acc, err := store.OpenAccount(qm.SenderAccount) acc, err := store.OpenAccount(log, qm.SenderAccount)
if err != nil { if err != nil {
return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err) return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
} }
@ -279,14 +280,14 @@ func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
defer func() { defer func() {
if dst != "" { if dst != "" {
err := os.Remove(dst) err := os.Remove(dst)
log.Check(err, "removing destination message file for queue", mlog.Field("path", dst)) log.Check(err, "removing destination message file for queue", slog.String("path", dst))
} }
}() }()
dstDir := filepath.Dir(dst) dstDir := filepath.Dir(dst)
os.MkdirAll(dstDir, 0770) os.MkdirAll(dstDir, 0770)
if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil { if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
return fmt.Errorf("linking/copying message to new file: %s", err) return fmt.Errorf("linking/copying message to new file: %s", err)
} else if err := moxio.SyncDir(dstDir); err != nil { } else if err := moxio.SyncDir(log, dstDir); err != nil {
return fmt.Errorf("sync directory: %v", err) return fmt.Errorf("sync directory: %v", err)
} }
@ -359,7 +360,7 @@ func Kick(ctx context.Context, ID int64, toDomain, recipient string, transport *
// Drop removes messages from the queue that match all nonzero parameters. // Drop removes messages from the queue that match all nonzero parameters.
// If all parameters are zero, all messages are removed. // If all parameters are zero, all messages are removed.
// Returns number of messages removed. // Returns number of messages removed.
func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) { func Drop(ctx context.Context, log mlog.Log, ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](ctx, DB) q := bstore.QueryDB[Msg](ctx, DB)
if ID > 0 { if ID > 0 {
q.FilterID(ID) q.FilterID(ID)
@ -381,7 +382,7 @@ func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int
for _, m := range msgs { for _, m := range msgs {
p := m.MessagePath() p := m.MessagePath()
if err := os.Remove(p); err != nil { if err := os.Remove(p); err != nil {
xlog.WithContext(ctx).Errorx("removing queue message from file system", err, mlog.Field("queuemsgid", m.ID), mlog.Field("path", p)) log.Errorx("removing queue message from file system", err, slog.Int64("queuemsgid", m.ID), slog.String("path", p))
} }
} }
return n, nil return n, nil
@ -427,6 +428,8 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
return err return err
} }
log := mlog.New("queue", nil)
// High-level delivery strategy advice: ../rfc/5321:3685 // High-level delivery strategy advice: ../rfc/5321:3685
go func() { go func() {
// Map keys are either dns.Domain.Name()'s, or string-formatted IP addresses. // Map keys are either dns.Domain.Name()'s, or string-formatted IP addresses.
@ -449,14 +452,14 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
continue continue
} }
launchWork(resolver, busyDomains) launchWork(log, resolver, busyDomains)
timer.Reset(nextWork(mox.Shutdown, busyDomains)) timer.Reset(nextWork(mox.Shutdown, log, busyDomains))
} }
}() }()
return nil return nil
} }
func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration { func nextWork(ctx context.Context, log mlog.Log, busyDomains map[string]struct{}) time.Duration {
q := bstore.QueryDB[Msg](ctx, DB) q := bstore.QueryDB[Msg](ctx, DB)
if len(busyDomains) > 0 { if len(busyDomains) > 0 {
var doms []any var doms []any
@ -471,13 +474,13 @@ func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duratio
if err == bstore.ErrAbsent { if err == bstore.ErrAbsent {
return 24 * time.Hour return 24 * time.Hour
} else if err != nil { } else if err != nil {
xlog.Errorx("finding time for next delivery attempt", err) log.Errorx("finding time for next delivery attempt", err)
return 1 * time.Minute return 1 * time.Minute
} }
return time.Until(qm.NextAttempt) return time.Until(qm.NextAttempt)
} }
func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int { func launchWork(log mlog.Log, resolver dns.Resolver, busyDomains map[string]struct{}) int {
q := bstore.QueryDB[Msg](mox.Shutdown, DB) q := bstore.QueryDB[Msg](mox.Shutdown, DB)
q.FilterLessEqual("NextAttempt", time.Now()) q.FilterLessEqual("NextAttempt", time.Now())
q.SortAsc("NextAttempt") q.SortAsc("NextAttempt")
@ -491,14 +494,14 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
} }
msgs, err := q.List() msgs, err := q.List()
if err != nil { if err != nil {
xlog.Errorx("querying for work in queue", err) log.Errorx("querying for work in queue", err)
mox.Sleep(mox.Shutdown, 1*time.Second) mox.Sleep(mox.Shutdown, 1*time.Second)
return -1 return -1
} }
for _, m := range msgs { for _, m := range msgs {
busyDomains[formatIPDomain(m.RecipientDomain)] = struct{}{} busyDomains[formatIPDomain(m.RecipientDomain)] = struct{}{}
go deliver(resolver, m) go deliver(log, resolver, m)
} }
return len(msgs) return len(msgs)
} }
@ -521,16 +524,15 @@ func queueDelete(ctx context.Context, msgID int64) error {
// deliver attempts to deliver a message. // deliver attempts to deliver a message.
// The queue is updated, either by removing a delivered or permanently failed // The queue is updated, either by removing a delivered or permanently failed
// message, or updating the time for the next attempt. A DSN may be sent. // message, or updating the time for the next attempt. A DSN may be sent.
func deliver(resolver dns.Resolver, m Msg) { func deliver(log mlog.Log, resolver dns.Resolver, m Msg) {
cid := mox.Cid() qlog := log.WithCid(mox.Cid()).With(slog.Any("from", m.Sender()), slog.Any("recipient", m.Recipient()), slog.Int("attempts", m.Attempts), slog.Int64("msgid", m.ID))
qlog := xlog.WithCid(cid).Fields(mlog.Field("from", m.Sender()), mlog.Field("recipient", m.Recipient()), mlog.Field("attempts", m.Attempts), mlog.Field("msgid", m.ID))
defer func() { defer func() {
deliveryResult <- formatIPDomain(m.RecipientDomain) deliveryResult <- formatIPDomain(m.RecipientDomain)
x := recover() x := recover()
if x != nil { if x != nil {
qlog.Error("deliver panic", mlog.Field("panic", x)) qlog.Error("deliver panic", slog.Any("panic", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Queue) metrics.PanicInc(metrics.Queue)
} }
@ -578,8 +580,8 @@ func deliver(resolver dns.Resolver, m Msg) {
} }
if transportName != "" { if transportName != "" {
qlog = qlog.Fields(mlog.Field("transport", transportName)) qlog = qlog.With(slog.String("transport", transportName))
qlog.Debug("delivering with transport", mlog.Field("transport", transportName)) qlog.Debug("delivering with transport")
} }
// We gather TLS connection successes and failures during delivery, and we store // We gather TLS connection successes and failures during delivery, and we store
@ -630,7 +632,7 @@ func deliver(resolver dns.Resolver, m Msg) {
// Ensure we store policy domain in unicode in database. // Ensure we store policy domain in unicode in database.
policyDomain, err := dns.ParseDomain(r.Policy.Domain) policyDomain, err := dns.ParseDomain(r.Policy.Domain)
if err != nil { if err != nil {
qlog.Errorx("parsing policy domain for tls result", err, mlog.Field("policydomain", r.Policy.Domain)) qlog.Errorx("parsing policy domain for tls result", err, slog.String("policydomain", r.Policy.Domain))
return return
} }
@ -667,12 +669,12 @@ func deliver(resolver dns.Resolver, m Msg) {
var dialer smtpclient.Dialer = &net.Dialer{} var dialer smtpclient.Dialer = &net.Dialer{}
if transport.Submissions != nil { if transport.Submissions != nil {
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465) deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465)
} else if transport.Submission != nil { } else if transport.Submission != nil {
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submission, false, 587) deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.Submission, false, 587)
} else if transport.SMTP != nil { } else if transport.SMTP != nil {
// todo future: perhaps also gather tlsrpt results for submissions. // todo future: perhaps also gather tlsrpt results for submissions.
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.SMTP, false, 25) deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.SMTP, false, 25)
} else { } else {
ourHostname := mox.Conf.Static.HostnameDomain ourHostname := mox.Conf.Static.HostnameDomain
if transport.Socks != nil { if transport.Socks != nil {
@ -688,7 +690,7 @@ func deliver(resolver dns.Resolver, m Msg) {
} }
ourHostname = transport.Socks.Hostname ourHostname = transport.Socks.Hostname
} }
recipientDomainResult, hostResults = deliverDirect(cid, qlog, resolver, dialer, ourHostname, transportName, m, backoff) recipientDomainResult, hostResults = deliverDirect(qlog, resolver, dialer, ourHostname, transportName, m, backoff)
} }
} }

59
queue/queue_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/smtpclient" "github.com/mjl-/mox/smtpclient"
@ -32,6 +33,7 @@ import (
) )
var ctxbg = context.Background() var ctxbg = context.Background()
var pkglog = mlog.New("queue", nil)
func tcheck(t *testing.T, err error, msg string) { func tcheck(t *testing.T, err error, msg string) {
if err != nil { if err != nil {
@ -61,12 +63,13 @@ func setup(t *testing.T) (*store.Account, func()) {
os.RemoveAll("../testdata/queue/data/queue") os.RemoveAll("../testdata/queue/data/queue")
} }
log := mlog.New("queue", nil)
mox.Context = ctxbg mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/queue/mox.conf") mox.ConfigStaticPath = filepath.FromSlash("../testdata/queue/mox.conf")
mox.MustLoadConfig(true, false) mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl") acc, err := store.OpenAccount(log, "mjl")
tcheck(t, err, "open account") tcheck(t, err, "open account")
err = acc.SetPassword("testtest") err = acc.SetPassword(log, "testtest")
tcheck(t, err, "set password") tcheck(t, err, "set password")
switchStop := store.Switchboard() switchStop := store.Switchboard()
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg) mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
@ -88,7 +91,7 @@ test email
func prepareFile(t *testing.T) *os.File { func prepareFile(t *testing.T) *os.File {
t.Helper() t.Helper()
msgFile, err := store.CreateMessageTemp("queue") msgFile, err := store.CreateMessageTemp(pkglog, "queue")
tcheck(t, err, "create temp message for delivery to queue") tcheck(t, err, "create temp message for delivery to queue")
_, err = msgFile.Write([]byte(testmsg)) _, err = msgFile.Write([]byte(testmsg))
tcheck(t, err, "write message file") tcheck(t, err, "write message file")
@ -115,11 +118,11 @@ func TestQueue(t *testing.T) {
var qm Msg var qm Msg
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg) msgs, err = List(ctxbg)
@ -131,7 +134,7 @@ func TestQueue(t *testing.T) {
if msg.Attempts != 0 { if msg.Attempts != 0 {
t.Fatalf("msg attempts %d, expected 0", msg.Attempts) t.Fatalf("msg attempts %d, expected 0", msg.Attempts)
} }
n, err := Drop(ctxbg, msgs[1].ID, "", "") n, err := Drop(ctxbg, pkglog, msgs[1].ID, "", "")
tcheck(t, err, "drop") tcheck(t, err, "drop")
if n != 1 { if n != 1 {
t.Fatalf("dropped %d, expected 1", n) t.Fatalf("dropped %d, expected 1", n)
@ -140,15 +143,15 @@ func TestQueue(t *testing.T) {
t.Fatalf("dropped message not removed from file system") t.Fatalf("dropped message not removed from file system")
} }
next := nextWork(ctxbg, nil) next := nextWork(ctxbg, pkglog, nil)
if next > 0 { if next > 0 {
t.Fatalf("nextWork in %s, should be now", next) t.Fatalf("nextWork in %s, should be now", next)
} }
busy := map[string]struct{}{"mox.example": {}} busy := map[string]struct{}{"mox.example": {}}
if x := nextWork(ctxbg, busy); x != 24*time.Hour { if x := nextWork(ctxbg, pkglog, busy); x != 24*time.Hour {
t.Fatalf("nextWork in %s for busy domain, should be in 24 hours", x) t.Fatalf("nextWork in %s for busy domain, should be in 24 hours", x)
} }
if nn := launchWork(nil, busy); nn != 0 { if nn := launchWork(pkglog, nil, busy); nn != 0 {
t.Fatalf("launchWork launched %d deliveries, expected 0", nn) t.Fatalf("launchWork launched %d deliveries, expected 0", nn)
} }
@ -171,7 +174,7 @@ func TestQueue(t *testing.T) {
smtpclient.DialHook = nil smtpclient.DialHook = nil
}() }()
launchWork(resolver, map[string]struct{}{}) launchWork(pkglog, resolver, map[string]struct{}{})
moxCert := fakeCert(t, "mail.mox.example", false) moxCert := fakeCert(t, "mail.mox.example", false)
@ -427,7 +430,7 @@ func TestQueue(t *testing.T) {
<-deliveryResult // Deliver sends here. <-deliveryResult // Deliver sends here.
} }
launchWork(resolver, map[string]struct{}{}) launchWork(pkglog, resolver, map[string]struct{}{})
waitDeliver() waitDeliver()
return wasNetDialer return wasNetDialer
} }
@ -449,7 +452,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of its route. // Add a message to be delivered with submit because of its route.
topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}} topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}}
qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
wasNetDialer = testDeliver(fakeSubmitServer) wasNetDialer = testDeliver(fakeSubmitServer)
if !wasNetDialer { if !wasNetDialer {
@ -458,7 +461,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of explicitly configured transport, that uses TLS. // Add a message to be delivered with submit because of explicitly configured transport, that uses TLS.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
transportSubmitTLS := "submittls" transportSubmitTLS := "submittls"
n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS) n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS)
@ -507,7 +510,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with socks. // Add a message to be delivered with socks.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
transportSocks := "socks" transportSocks := "socks"
n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks) n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks)
@ -523,7 +526,7 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with opportunistic TLS verification. // Add message to be delivered with opportunistic TLS verification.
clearTLSResults(t) clearTLSResults(t)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -537,7 +540,7 @@ func TestQueue(t *testing.T) {
// Test fallback to plain text with TLS handshake fails. // Test fallback to plain text with TLS handshake fails.
clearTLSResults(t) clearTLSResults(t)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -557,7 +560,7 @@ func TestQueue(t *testing.T) {
}, },
} }
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -578,7 +581,7 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with verified TLS and REQUIRETLS. // Add message to be delivered with verified TLS and REQUIRETLS.
yes := true yes := true
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -595,7 +598,7 @@ func TestQueue(t *testing.T) {
}, },
} }
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -616,7 +619,7 @@ func TestQueue(t *testing.T) {
}, },
} }
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -638,7 +641,7 @@ func TestQueue(t *testing.T) {
// Check that message is delivered with TLS-Required: No and non-matching DANE record. // Check that message is delivered with TLS-Required: No and non-matching DANE record.
no := false no := false
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -649,7 +652,7 @@ func TestQueue(t *testing.T) {
// Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text. // Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -660,7 +663,7 @@ func TestQueue(t *testing.T) {
// Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers. // Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -675,7 +678,7 @@ func TestQueue(t *testing.T) {
// Add message with requiretls that fails immediately due to no verification policy for recipient domain. // Add message with requiretls that fails immediately due to no verification policy for recipient domain.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
@ -690,7 +693,7 @@ func TestQueue(t *testing.T) {
// Add another message that we'll fail to deliver entirely. // Add another message that we'll fail to deliver entirely.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg) msgs, err = List(ctxbg)
@ -756,7 +759,7 @@ func TestQueue(t *testing.T) {
resolver.AllAuthentic = false resolver.AllAuthentic = false
resolver.TLSA = nil resolver.TLSA = nil
} }
deliver(resolver, msg) deliver(pkglog, resolver, msg)
err = DB.Get(ctxbg, &msg) err = DB.Get(ctxbg, &msg)
tcheck(t, err, "get msg") tcheck(t, err, "get msg")
if msg.Attempts != i { if msg.Attempts != i {
@ -779,7 +782,7 @@ func TestQueue(t *testing.T) {
// Trigger final failure. // Trigger final failure.
go func() { <-deliveryResult }() // Deliver sends here. go func() { <-deliveryResult }() // Deliver sends here.
deliver(resolver, msg) deliver(pkglog, resolver, msg)
err = DB.Get(ctxbg, &msg) err = DB.Get(ctxbg, &msg)
if err != bstore.ErrAbsent { if err != bstore.ErrAbsent {
t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err) t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err)
@ -881,7 +884,7 @@ func TestQueueStart(t *testing.T) {
defer os.Remove(mf.Name()) defer os.Remove(mf.Name())
defer mf.Close() defer mf.Close()
qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf) err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
checkDialed(true) checkDialed(true)

22
queue/submit.go
View file

@ -10,6 +10,8 @@ import (
"os" "os"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/config" "github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dsn" "github.com/mjl-/mox/dsn"
@ -25,7 +27,7 @@ import (
// deliver via another SMTP server, e.g. relaying to a smart host, possibly // deliver via another SMTP server, e.g. relaying to a smart host, possibly
// with authentication (submission). // with authentication (submission).
func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) { func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
// todo: configurable timeouts // todo: configurable timeouts
port := transport.Port port := transport.Port
@ -52,7 +54,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
var success bool var success bool
defer func() { defer func() {
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, string(tlsMode), deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second)) metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, string(tlsMode), deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second))
qlog.Debug("queue deliversubmit result", mlog.Field("host", transport.DNSHost), mlog.Field("port", port), mlog.Field("attempt", m.Attempts), mlog.Field("permanent", permanent), mlog.Field("secodeopt", secodeOpt), mlog.Field("errmsg", errmsg), mlog.Field("ok", success), mlog.Field("duration", time.Since(start))) qlog.Debug("queue deliversubmit result", slog.Any("host", transport.DNSHost), slog.Int("port", port), slog.Int("attempt", m.Attempts), slog.Bool("permanent", permanent), slog.String("secodeopt", secodeOpt), slog.String("errmsg", errmsg), slog.Bool("ok", success), slog.Duration("duration", time.Since(start)))
}() }()
// todo: SMTP-DANE should be used when relaying on port 25. // todo: SMTP-DANE should be used when relaying on port 25.
@ -74,13 +76,13 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
if m.DialedIPs == nil { if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{} m.DialedIPs = map[string][]net.IP{}
} }
_, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs) _, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog.Logger, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs)
var conn net.Conn var conn net.Conn
if err == nil { if err == nil {
if m.DialedIPs == nil { if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{} m.DialedIPs = map[string][]net.IP{}
} }
conn, _, err = smtpclient.Dial(dialctx, qlog, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs) conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs)
} }
addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port)) addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port))
var result string var result string
@ -100,7 +102,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
err := conn.Close() err := conn.Close()
qlog.Check(err, "closing connection") qlog.Check(err, "closing connection")
} }
qlog.Errorx("dialing for submission", err, mlog.Field("remote", addr)) qlog.Errorx("dialing for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err) errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return return
@ -122,7 +124,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
auth = append(auth, sasl.NewClientSCRAMSHA256(a.Username, a.Password)) auth = append(auth, sasl.NewClientSCRAMSHA256(a.Username, a.Password))
default: default:
// Should not happen. // Should not happen.
qlog.Error("missing smtp authentication mechanisms implementation", mlog.Field("mechanism", mech)) qlog.Error("missing smtp authentication mechanisms implementation", slog.String("mechanism", mech))
errmsg = fmt.Sprintf("transport %s: authentication mechanisms %q not implemented", transportName, mech) errmsg = fmt.Sprintf("transport %s: authentication mechanisms %q not implemented", transportName, mech)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return return
@ -135,14 +137,14 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
Auth: auth, Auth: auth,
RootCAs: mox.Conf.Static.TLS.CertPool, RootCAs: mox.Conf.Static.TLS.CertPool,
} }
client, err := smtpclient.New(clientctx, qlog, conn, tlsMode, tlsPKIX, mox.Conf.Static.HostnameDomain, transport.DNSHost, opts) client, err := smtpclient.New(clientctx, qlog.Logger, conn, tlsMode, tlsPKIX, mox.Conf.Static.HostnameDomain, transport.DNSHost, opts)
if err != nil { if err != nil {
smtperr, ok := err.(smtpclient.Error) smtperr, ok := err.(smtpclient.Error)
var remoteMTA dsn.NameIP var remoteMTA dsn.NameIP
if ok { if ok {
remoteMTA.Name = transport.Host remoteMTA.Name = transport.Host
} }
qlog.Errorx("establishing smtp session for submission", err, mlog.Field("remote", addr)) qlog.Errorx("establishing smtp session for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err) errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err)
secodeOpt = smtperr.Secode secodeOpt = smtperr.Secode
fail(qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg) fail(qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg)
@ -168,7 +170,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
p := m.MessagePath() p := m.MessagePath()
f, err := os.Open(p) f, err := os.Open(p)
if err != nil { if err != nil {
qlog.Errorx("opening message for delivery", err, mlog.Field("remote", addr), mlog.Field("path", p)) qlog.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p))
errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err) errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return return
@ -209,7 +211,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
if ok { if ok {
remoteMTA.Name = transport.Host remoteMTA.Name = transport.Host
} }
qlog.Errorx("submitting email", err, mlog.Field("remote", addr)) qlog.Errorx("submitting email", err, slog.String("remote", addr))
permanent = smtperr.Permanent permanent = smtperr.Permanent
secodeOpt = smtperr.Secode secodeOpt = smtperr.Secode
errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err) errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err)

8
quickstart.go
View file

@ -535,7 +535,7 @@ messages over SMTP.
for _, zone := range zones { for _, zone := range zones {
for _, ip := range hostIPs { for _, ip := range hostIPs {
dnsblctx, dnsblcancel := context.WithTimeout(resolveCtx, 5*time.Second) dnsblctx, dnsblcancel := context.WithTimeout(resolveCtx, 5*time.Second)
status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(ip)) status, expl, err := dnsbl.Lookup(dnsblctx, c.log.Logger, resolver, zone, net.ParseIP(ip))
dnsblcancel() dnsblcancel()
if status == dnsbl.StatusPass { if status == dnsbl.StatusPass {
continue continue
@ -813,7 +813,7 @@ and check the admin page for the needed DNS records.`)
// Verify config. // Verify config.
loadTLSKeyCerts := !existingWebserver loadTLSKeyCerts := !existingWebserver
mc, errs := mox.ParseConfig(context.Background(), filepath.FromSlash("config/mox.conf"), true, loadTLSKeyCerts, false) mc, errs := mox.ParseConfig(context.Background(), c.log, filepath.FromSlash("config/mox.conf"), true, loadTLSKeyCerts, false)
if len(errs) > 0 { if len(errs) > 0 {
if len(errs) > 1 { if len(errs) > 1 {
log.Printf("checking generated config, multiple errors:") log.Printf("checking generated config, multiple errors:")
@ -834,14 +834,14 @@ and check the admin page for the needed DNS records.`)
fatalf("cannot find domain in new config") fatalf("cannot find domain in new config")
} }
acc, _, err := store.OpenEmail(args[0]) acc, _, err := store.OpenEmail(c.log, args[0])
if err != nil { if err != nil {
fatalf("open account: %s", err) fatalf("open account: %s", err)
} }
cleanupPaths = append(cleanupPaths, dataDir, filepath.Join(dataDir, "accounts"), filepath.Join(dataDir, "accounts", accountName), filepath.Join(dataDir, "accounts", accountName, "index.db")) cleanupPaths = append(cleanupPaths, dataDir, filepath.Join(dataDir, "accounts"), filepath.Join(dataDir, "accounts", accountName), filepath.Join(dataDir, "accounts", accountName, "index.db"))
password := pwgen() password := pwgen()
if err := acc.SetPassword(password); err != nil { if err := acc.SetPassword(c.log, password); err != nil {
fatalf("setting password: %s", err) fatalf("setting password: %s", err)
} }
if err := acc.Close(); err != nil { if err := acc.Close(); err != nil {

3
sendmail.go
View file

@ -17,7 +17,6 @@ import (
"github.com/mjl-/sconf" "github.com/mjl-/sconf"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/sasl" "github.com/mjl-/mox/sasl"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
@ -300,7 +299,7 @@ binary should be setgid that group:
Auth: auth, Auth: auth,
RootCAs: mox.Conf.Static.TLS.CertPool, RootCAs: mox.Conf.Static.TLS.CertPool,
} }
client, err := smtpclient.New(ctx, mlog.New("sendmail"), conn, tlsMode, tlsPKIX, ourHostname, remoteHostname, opts) client, err := smtpclient.New(ctx, c.log.Logger, conn, tlsMode, tlsPKIX, ourHostname, remoteHostname, opts)
xsavecheckf(err, "open smtp session") xsavecheckf(err, "open smtp session")
err = client.Deliver(ctx, submitconf.From, recipient, int64(len(msg)), strings.NewReader(msg), true, false, submitconf.RequireTLS == RequireTLSYes) err = client.Deliver(ctx, submitconf.From, recipient, int64(len(msg)), strings.NewReader(msg), true, false, submitconf.RequireTLS == RequireTLSYes)

2
serve.go
View file

@ -19,7 +19,7 @@ import (
"github.com/mjl-/mox/tlsrptsend" "github.com/mjl-/mox/tlsrptsend"
) )
func shutdown(log *mlog.Log) { func shutdown(log mlog.Log) {
// We indicate we are shutting down. Causes new connections and new SMTP commands // We indicate we are shutting down. Causes new connections and new SMTP commands
// to be rejected. Should stop active connections pretty quickly. // to be rejected. Should stop active connections pretty quickly.
mox.ShutdownCancel() mox.ShutdownCancel()

70
serve_unix.go
View file

@ -18,6 +18,8 @@ import (
"syscall" "syscall"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -32,12 +34,12 @@ import (
"github.com/mjl-/mox/updates" "github.com/mjl-/mox/updates"
) )
func monitorDNSBL(log *mlog.Log) { func monitorDNSBL(log mlog.Log) {
defer func() { defer func() {
// On error, don't bring down the entire server. // On error, don't bring down the entire server.
x := recover() x := recover()
if x != nil { if x != nil {
log.Error("monitordnsbl panic", mlog.Field("panic", x)) log.Error("monitordnsbl panic", slog.Any("panic", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Serve) metrics.PanicInc(metrics.Serve)
} }
@ -53,7 +55,7 @@ func monitorDNSBL(log *mlog.Log) {
for _, zone := range l.SMTP.DNSBLs { for _, zone := range l.SMTP.DNSBLs {
d, err := dns.ParseDomain(zone) d, err := dns.ParseDomain(zone)
if err != nil { if err != nil {
log.Fatalx("parsing dnsbls zone", err, mlog.Field("zone", zone)) log.Fatalx("parsing dnsbls zone", err, slog.Any("zone", zone))
} }
zones = append(zones, d) zones = append(zones, d)
} }
@ -86,9 +88,9 @@ func monitorDNSBL(log *mlog.Log) {
} }
for _, zone := range zones { for _, zone := range zones {
status, expl, err := dnsbl.Lookup(mox.Context, resolver, zone, ip) status, expl, err := dnsbl.Lookup(mox.Context, log.Logger, resolver, zone, ip)
if err != nil { if err != nil {
log.Errorx("dnsbl monitor lookup", err, mlog.Field("ip", ip), mlog.Field("zone", zone), mlog.Field("expl", expl), mlog.Field("status", status)) log.Errorx("dnsbl monitor lookup", err, slog.Any("ip", ip), slog.Any("zone", zone), slog.String("expl", expl), slog.Any("status", status))
} }
k := key{zone, ip.String()} k := key{zone, ip.String()}
@ -145,7 +147,7 @@ Only implemented on unix systems, not Windows.
checkACMEHosts := os.Getuid() != 0 checkACMEHosts := os.Getuid() != 0
log := mlog.New("serve") log := c.log
if os.Getuid() == 0 { if os.Getuid() == 0 {
mox.MustLoadConfig(true, checkACMEHosts) mox.MustLoadConfig(true, checkACMEHosts)
@ -159,7 +161,7 @@ Only implemented on unix systems, not Windows.
domainsconf, err := filepath.Abs(mox.ConfigDynamicPath) domainsconf, err := filepath.Abs(mox.ConfigDynamicPath)
log.Check(err, "finding absolute domains.conf path") log.Check(err, "finding absolute domains.conf path")
log.Print("starting as root, initializing network listeners", mlog.Field("version", moxvar.Version), mlog.Field("pid", os.Getpid()), mlog.Field("moxconf", moxconf), mlog.Field("domainsconf", domainsconf)) log.Print("starting as root, initializing network listeners", slog.String("version", moxvar.Version), slog.Any("pid", os.Getpid()), slog.String("moxconf", moxconf), slog.String("domainsconf", domainsconf))
if os.Getenv("MOX_SOCKETS") != "" { if os.Getenv("MOX_SOCKETS") != "" {
log.Fatal("refusing to start as root with $MOX_SOCKETS set") log.Fatal("refusing to start as root with $MOX_SOCKETS set")
} }
@ -185,7 +187,7 @@ Only implemented on unix systems, not Windows.
} else { } else {
mox.RestorePassedFiles() mox.RestorePassedFiles()
mox.MustLoadConfig(true, checkACMEHosts) mox.MustLoadConfig(true, checkACMEHosts)
log.Print("starting as unprivileged user", mlog.Field("user", mox.Conf.Static.User), mlog.Field("uid", mox.Conf.Static.UID), mlog.Field("gid", mox.Conf.Static.GID), mlog.Field("pid", os.Getpid())) log.Print("starting as unprivileged user", slog.String("user", mox.Conf.Static.User), slog.Any("uid", mox.Conf.Static.UID), slog.Any("gid", mox.Conf.Static.GID), slog.Any("pid", os.Getpid()))
} }
syscall.Umask(syscall.Umask(007) | 007) syscall.Umask(syscall.Umask(007) | 007)
@ -200,12 +202,12 @@ Only implemented on unix systems, not Windows.
log.Fatalx("reading random recvid data", err) log.Fatalx("reading random recvid data", err)
} }
if err := os.WriteFile(recvidpath, recvidbuf, 0660); err != nil { if err := os.WriteFile(recvidpath, recvidbuf, 0660); err != nil {
log.Fatalx("writing recvidpath", err, mlog.Field("path", recvidpath)) log.Fatalx("writing recvidpath", err, slog.String("path", recvidpath))
} }
err := os.Chown(recvidpath, int(mox.Conf.Static.UID), 0) err := os.Chown(recvidpath, int(mox.Conf.Static.UID), 0)
log.Check(err, "chown receveidid.key", mlog.Field("path", recvidpath), mlog.Field("uid", mox.Conf.Static.UID), mlog.Field("gid", 0)) log.Check(err, "chown receveidid.key", slog.String("path", recvidpath), slog.Any("uid", mox.Conf.Static.UID), slog.Any("gid", 0))
err = os.Chmod(recvidpath, 0640) err = os.Chmod(recvidpath, 0640)
log.Check(err, "chmod receveidid.key to 0640", mlog.Field("path", recvidpath)) log.Check(err, "chmod receveidid.key to 0640", slog.String("path", recvidpath))
} }
if err := mox.ReceivedIDInit(recvidbuf[:16], recvidbuf[16:]); err != nil { if err := mox.ReceivedIDInit(recvidbuf[:16], recvidbuf[16:]); err != nil {
log.Fatalx("init receivedid", err) log.Fatalx("init receivedid", err)
@ -242,7 +244,7 @@ Only implemented on unix systems, not Windows.
// mtime. But file won't exist initially. // mtime. But file won't exist initially.
if !mtime.IsZero() && time.Since(mtime) < 24*time.Hour { if !mtime.IsZero() && time.Since(mtime) < 24*time.Hour {
d := 24*time.Hour - time.Since(mtime) d := 24*time.Hour - time.Since(mtime)
log.Debug("sleeping for next check for updates", mlog.Field("sleep", d)) log.Debug("sleeping for next check for updates", slog.Duration("sleep", d))
time.Sleep(d) time.Sleep(d)
next = 0 next = 0
} }
@ -253,12 +255,12 @@ Only implemented on unix systems, not Windows.
} }
} }
log.Debug("checking for updates", mlog.Field("lastknown", lastknown)) log.Debug("checking for updates", slog.Any("lastknown", lastknown))
updatesctx, updatescancel := context.WithTimeout(mox.Context, time.Minute) updatesctx, updatescancel := context.WithTimeout(mox.Context, time.Minute)
latest, _, changelog, err := updates.Check(updatesctx, dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain}, lastknown, changelogURL, changelogPubKey) latest, _, changelog, err := updates.Check(updatesctx, log.Logger, dns.StrictResolver{Log: log.Logger}, dns.Domain{ASCII: changelogDomain}, lastknown, changelogURL, changelogPubKey)
updatescancel() updatescancel()
if err != nil { if err != nil {
log.Infox("checking for updates", err, mlog.Field("latest", latest)) log.Infox("checking for updates", err, slog.Any("latest", latest))
return next return next
} }
if !latest.After(lastknown) { if !latest.After(lastknown) {
@ -266,7 +268,7 @@ Only implemented on unix systems, not Windows.
return next return next
} }
if len(changelog.Changes) == 0 { if len(changelog.Changes) == 0 {
log.Info("new version available, but changelog is empty, ignoring", mlog.Field("latest", latest)) log.Info("new version available, but changelog is empty, ignoring", slog.Any("latest", latest))
return next return next
} }
@ -276,7 +278,7 @@ Only implemented on unix systems, not Windows.
} }
cl += "----" cl += "----"
a, err := store.OpenAccount(mox.Conf.Static.Postmaster.Account) a, err := store.OpenAccount(log, mox.Conf.Static.Postmaster.Account)
if err != nil { if err != nil {
log.Infox("open account for postmaster changelog delivery", err) log.Infox("open account for postmaster changelog delivery", err)
return next return next
@ -285,7 +287,7 @@ Only implemented on unix systems, not Windows.
err := a.Close() err := a.Close()
log.Check(err, "closing account") log.Check(err, "closing account")
}() }()
f, err := store.CreateMessageTemp("changelog") f, err := store.CreateMessageTemp(log, "changelog")
if err != nil { if err != nil {
log.Infox("making temporary message file for changelog delivery", err) log.Infox("making temporary message file for changelog delivery", err)
return next return next
@ -306,7 +308,7 @@ Only implemented on unix systems, not Windows.
log.Errorx("changelog delivery", err) log.Errorx("changelog delivery", err)
return next return next
} }
log.Info("delivered changelog", mlog.Field("current", current), mlog.Field("lastknown", lastknown), mlog.Field("latest", latest)) log.Info("delivered changelog", slog.Any("current", current), slog.Any("lastknown", lastknown), slog.Any("latest", latest))
if err := mox.StoreLastKnown(latest); err != nil { if err := mox.StoreLastKnown(latest); err != nil {
// This will be awkward, we'll keep notifying the postmaster once every 24h... // This will be awkward, we'll keep notifying the postmaster once every 24h...
log.Infox("updating last known version", err) log.Infox("updating last known version", err)
@ -353,13 +355,13 @@ Only implemented on unix systems, not Windows.
now := time.Now() now := time.Now()
for _, e := range tmps { for _, e := range tmps {
if fi, err := e.Info(); err != nil { if fi, err := e.Info(); err != nil {
log.Errorx("stat tmp file", err, mlog.Field("filename", e.Name())) log.Errorx("stat tmp file", err, slog.String("filename", e.Name()))
} else if now.Sub(fi.ModTime()) > 7*24*time.Hour && !fi.IsDir() { } else if now.Sub(fi.ModTime()) > 7*24*time.Hour && !fi.IsDir() {
p := filepath.Join(tmpdir, e.Name()) p := filepath.Join(tmpdir, e.Name())
if err := os.Remove(p); err != nil { if err := os.Remove(p); err != nil {
log.Errorx("removing stale temporary file", err, mlog.Field("path", p)) log.Errorx("removing stale temporary file", err, slog.String("path", p))
} else { } else {
log.Info("removed stale temporary file", mlog.Field("path", p)) log.Info("removed stale temporary file", slog.String("path", p))
} }
} }
} }
@ -369,7 +371,7 @@ Only implemented on unix systems, not Windows.
sigc := make(chan os.Signal, 1) sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
sig := <-sigc sig := <-sigc
log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig)) log.Print("shutting down, waiting max 3s for existing connections", slog.Any("signal", sig))
shutdown(log) shutdown(log)
if num, ok := sig.(syscall.Signal); ok { if num, ok := sig.(syscall.Signal); ok {
os.Exit(int(num)) os.Exit(int(num))
@ -383,7 +385,7 @@ Only implemented on unix systems, not Windows.
// We require being able to stat the basic non-optional paths. Then we'll try to // We require being able to stat the basic non-optional paths. Then we'll try to
// fix up permissions. If an error occurs when fixing permissions, we log and // fix up permissions. If an error occurs when fixing permissions, we log and
// continue (could not be an actual problem). // continue (could not be an actual problem).
func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid uint32) (rerr error) { func fixperms(log mlog.Log, workdir, configdir, datadir string, moxuid, moxgid uint32) (rerr error) {
type fserr struct{ Err error } type fserr struct{ Err error }
defer func() { defer func() {
x := recover() x := recover()
@ -483,33 +485,33 @@ func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid
for _, ch := range changes { for _, ch := range changes {
if ch.uid != nil { if ch.uid != nil {
err := os.Chown(ch.path, int(*ch.uid), int(*ch.gid)) err := os.Chown(ch.path, int(*ch.uid), int(*ch.gid))
log.Printx("chown, fixing uid/gid", err, mlog.Field("path", ch.path), mlog.Field("olduid", ch.olduid), mlog.Field("oldgid", ch.oldgid), mlog.Field("newuid", *ch.uid), mlog.Field("newgid", *ch.gid)) log.Printx("chown, fixing uid/gid", err, slog.String("path", ch.path), slog.Any("olduid", ch.olduid), slog.Any("oldgid", ch.oldgid), slog.Any("newuid", *ch.uid), slog.Any("newgid", *ch.gid))
} }
if ch.mode != nil { if ch.mode != nil {
err := os.Chmod(ch.path, *ch.mode) err := os.Chmod(ch.path, *ch.mode)
log.Printx("chmod, fixing permissions", err, mlog.Field("path", ch.path), mlog.Field("oldmode", fmt.Sprintf("%03o", ch.oldmode)), mlog.Field("newmode", fmt.Sprintf("%03o", *ch.mode))) log.Printx("chmod, fixing permissions", err, slog.String("path", ch.path), slog.Any("oldmode", fmt.Sprintf("%03o", ch.oldmode)), slog.Any("newmode", fmt.Sprintf("%03o", *ch.mode)))
} }
} }
walkchange := func(dir string) { walkchange := func(dir string) {
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
log.Printx("walk error, continuing", err, mlog.Field("path", path)) log.Printx("walk error, continuing", err, slog.String("path", path))
return nil return nil
} }
fi, err := d.Info() fi, err := d.Info()
if err != nil { if err != nil {
log.Printx("stat during walk, continuing", err, mlog.Field("path", path)) log.Printx("stat during walk, continuing", err, slog.String("path", path))
return nil return nil
} }
st, ok := fi.Sys().(*syscall.Stat_t) st, ok := fi.Sys().(*syscall.Stat_t)
if !ok { if !ok {
log.Printx("syscall stat during walk, continuing", err, mlog.Field("path", path)) log.Printx("syscall stat during walk, continuing", err, slog.String("path", path))
return nil return nil
} }
if st.Uid != moxuid || st.Gid != root { if st.Uid != moxuid || st.Gid != root {
err := os.Chown(path, int(moxuid), root) err := os.Chown(path, int(moxuid), root)
log.Printx("walk chown, fixing uid/gid", err, mlog.Field("path", path), mlog.Field("olduid", st.Uid), mlog.Field("oldgid", st.Gid), mlog.Field("newuid", moxuid), mlog.Field("newgid", root)) log.Printx("walk chown, fixing uid/gid", err, slog.String("path", path), slog.Any("olduid", st.Uid), slog.Any("oldgid", st.Gid), slog.Any("newuid", moxuid), slog.Any("newgid", root))
} }
omode := fi.Mode() & (fs.ModeSetgid | 0777) omode := fi.Mode() & (fs.ModeSetgid | 0777)
var nmode fs.FileMode var nmode fs.FileMode
@ -520,21 +522,21 @@ func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid
} }
if omode != nmode { if omode != nmode {
err := os.Chmod(path, nmode) err := os.Chmod(path, nmode)
log.Printx("walk chmod, fixing permissions", err, mlog.Field("path", path), mlog.Field("oldmode", fmt.Sprintf("%03o", omode)), mlog.Field("newmode", fmt.Sprintf("%03o", nmode))) log.Printx("walk chmod, fixing permissions", err, slog.String("path", path), slog.Any("oldmode", fmt.Sprintf("%03o", omode)), slog.Any("newmode", fmt.Sprintf("%03o", nmode)))
} }
return nil return nil
}) })
log.Check(err, "walking dir to fix permissions", mlog.Field("dir", dir)) log.Check(err, "walking dir to fix permissions", slog.String("dir", dir))
} }
// If config or data dir needed fixing, also set uid/gid and mode and files/dirs // If config or data dir needed fixing, also set uid/gid and mode and files/dirs
// inside, recursively. We don't always recurse, data probably contains many files. // inside, recursively. We don't always recurse, data probably contains many files.
if fixconfig { if fixconfig {
log.Print("fixing permissions in config dir", mlog.Field("configdir", configdir)) log.Print("fixing permissions in config dir", slog.String("configdir", configdir))
walkchange(configdir) walkchange(configdir)
} }
if fixdata { if fixdata {
log.Print("fixing permissions in data dir", mlog.Field("configdir", configdir)) log.Print("fixing permissions in data dir", slog.String("configdir", configdir))
walkchange(datadir) walkchange(datadir)
} }
return nil return nil

46
smtpclient/client.go
View file

@ -16,6 +16,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -118,7 +120,7 @@ type Client struct {
w *bufio.Writer w *bufio.Writer
tr *moxio.TraceReader // Kept for changing trace levels between cmd/auth/data. tr *moxio.TraceReader // Kept for changing trace levels between cmd/auth/data.
tw *moxio.TraceWriter tw *moxio.TraceWriter
log *mlog.Log log mlog.Log
lastlog time.Time // For adding delta timestamps between log lines. lastlog time.Time // For adding delta timestamps between log lines.
cmds []string // Last or active command, for generating errors and metrics. cmds []string // Last or active command, for generating errors and metrics.
cmdStart time.Time // Start of command. cmdStart time.Time // Start of command.
@ -237,7 +239,7 @@ type Opts struct {
// with opportunistic TLS without PKIX verification by default. Recipient domains // with opportunistic TLS without PKIX verification by default. Recipient domains
// can opt-in to PKIX verification by publishing an MTA-STS policy, or opt-in to // can opt-in to PKIX verification by publishing an MTA-STS policy, or opt-in to
// DANE verification by publishing DNSSEC-protected TLSA records in DNS. // DANE verification by publishing DNSSEC-protected TLSA records in DNS.
func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tlsVerifyPKIX bool, ehloHostname, remoteHostname dns.Domain, opts Opts) (*Client, error) { func New(ctx context.Context, elog *slog.Logger, conn net.Conn, tlsMode TLSMode, tlsVerifyPKIX bool, ehloHostname, remoteHostname dns.Domain, opts Opts) (*Client, error) {
ensureResult := func(r *tlsrpt.Result) *tlsrpt.Result { ensureResult := func(r *tlsrpt.Result) *tlsrpt.Result {
if r == nil { if r == nil {
return &tlsrpt.Result{} return &tlsrpt.Result{}
@ -259,10 +261,10 @@ func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tls
recipientDomainResult: ensureResult(opts.RecipientDomainResult), recipientDomainResult: ensureResult(opts.RecipientDomainResult),
hostResult: ensureResult(opts.HostResult), hostResult: ensureResult(opts.HostResult),
} }
c.log = log.Fields(mlog.Field("smtpclient", "")).MoreFields(func() []mlog.Pair { c.log = mlog.New("smtpclient", elog).WithFunc(func() []slog.Attr {
now := time.Now() now := time.Now()
l := []mlog.Pair{ l := []slog.Attr{
mlog.Field("delta", now.Sub(c.lastlog)), slog.Duration("delta", now.Sub(c.lastlog)),
} }
c.lastlog = now c.lastlog = now
return l return l
@ -280,7 +282,7 @@ func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tls
c.tlsResultAdd(1, 0, nil) c.tlsResultAdd(1, 0, nil)
c.conn = tlsconn c.conn = tlsconn
tlsversion, ciphersuite := mox.TLSInfo(tlsconn) tlsversion, ciphersuite := mox.TLSInfo(tlsconn)
c.log.Debug("tls client handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite), mlog.Field("servername", remoteHostname)) c.log.Debug("tls client handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite), slog.Any("servername", remoteHostname))
c.tls = true c.tls = true
} else { } else {
c.conn = conn c.conn = conn
@ -329,8 +331,8 @@ func (c *Client) tlsConfig() *tls.Config {
// DANE verification. // DANE verification.
// daneRecords can be non-nil and empty, that's intended. // daneRecords can be non-nil and empty, that's intended.
if c.daneRecords != nil { if c.daneRecords != nil {
verified, record, err := dane.Verify(c.log, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames) verified, record, err := dane.Verify(c.log.Logger, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames)
c.log.Debugx("dane verification", err, mlog.Field("verified", verified), mlog.Field("record", record)) c.log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record))
if verified { if verified {
if c.daneVerifiedRecord != nil { if c.daneVerifiedRecord != nil {
*c.daneVerifiedRecord = record *c.daneVerifiedRecord = record
@ -426,7 +428,7 @@ func (c *Client) xerrorf(permanent bool, code int, secode, lastLine, format stri
type timeoutWriter struct { type timeoutWriter struct {
conn net.Conn conn net.Conn
timeout time.Duration timeout time.Duration
log *mlog.Log log mlog.Log
} }
func (w timeoutWriter) Write(buf []byte) (int, error) { func (w timeoutWriter) Write(buf []byte) (int, error) {
@ -445,7 +447,7 @@ func (c *Client) readline() (string, error) {
c.log.Errorx("setting read deadline", err) c.log.Errorx("setting read deadline", err)
} }
line, err := bufs.Readline(c.r) line, err := bufs.Readline(c.log, c.r)
if err != nil { if err != nil {
// See if this is a TLS alert from remote, and one other than 0 (which notifies // See if this is a TLS alert from remote, and one other than 0 (which notifies
// that the connection is being closed. If so, we register a TLS connection // that the connection is being closed. If so, we register a TLS connection
@ -463,7 +465,7 @@ func (c *Client) readline() (string, error) {
return line, nil return line, nil
} }
func (c *Client) xtrace(level mlog.Level) func() { func (c *Client) xtrace(level slog.Level) func() {
c.xflush() c.xflush()
c.tr.SetTrace(level) c.tr.SetTrace(level)
c.tw.SetTrace(level) c.tw.SetTrace(level)
@ -543,7 +545,7 @@ func (c *Client) readecode(ecodes bool) (code int, secode, lastLine string, text
} }
} }
metricCommands.WithLabelValues(cmd, fmt.Sprintf("%d", co), sec).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) metricCommands.WithLabelValues(cmd, fmt.Sprintf("%d", co), sec).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
c.log.Debug("smtpclient command result", mlog.Field("cmd", cmd), mlog.Field("code", co), mlog.Field("secode", sec), mlog.Field("duration", time.Since(c.cmdStart))) c.log.Debug("smtpclient command result", slog.String("cmd", cmd), slog.Int("code", co), slog.String("secode", sec), slog.Duration("duration", time.Since(c.cmdStart)))
} }
return co, sec, line, texts, nil return co, sec, line, texts, nil
} }
@ -726,7 +728,7 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do
// Attempt TLS if remote understands STARTTLS and we aren't doing immediate TLS or if caller requires it. // Attempt TLS if remote understands STARTTLS and we aren't doing immediate TLS or if caller requires it.
if c.extStartTLS && tlsMode == TLSOpportunistic || tlsMode == TLSRequiredStartTLS { if c.extStartTLS && tlsMode == TLSOpportunistic || tlsMode == TLSRequiredStartTLS {
c.log.Debug("starting tls client", mlog.Field("tlsmode", tlsMode), mlog.Field("servername", c.remoteHostname)) c.log.Debug("starting tls client", slog.Any("tlsmode", tlsMode), slog.Any("servername", c.remoteHostname))
c.cmds[0] = "starttls" c.cmds[0] = "starttls"
c.cmdStart = time.Now() c.cmdStart = time.Now()
c.xwritelinef("STARTTLS") c.xwritelinef("STARTTLS")
@ -772,14 +774,14 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do
tlsversion, ciphersuite := mox.TLSInfo(nconn) tlsversion, ciphersuite := mox.TLSInfo(nconn)
c.log.Debug("starttls client handshake done", c.log.Debug("starttls client handshake done",
mlog.Field("tlsmode", tlsMode), slog.Any("tlsmode", tlsMode),
mlog.Field("verifypkix", c.tlsVerifyPKIX), slog.Bool("verifypkix", c.tlsVerifyPKIX),
mlog.Field("verifydane", c.daneRecords != nil), slog.Bool("verifydane", c.daneRecords != nil),
mlog.Field("ignoretlsverifyerrors", c.ignoreTLSVerifyErrors), slog.Bool("ignoretlsverifyerrors", c.ignoreTLSVerifyErrors),
mlog.Field("tls", tlsversion), slog.String("tls", tlsversion),
mlog.Field("ciphersuite", ciphersuite), slog.String("ciphersuite", ciphersuite),
mlog.Field("servername", c.remoteHostname), slog.Any("servername", c.remoteHostname),
mlog.Field("danerecord", c.daneVerifiedRecord)) slog.Any("danerecord", c.daneVerifiedRecord))
c.tls = true c.tls = true
// Track successful TLS connection. ../rfc/8460:515 // Track successful TLS connection. ../rfc/8460:515
c.tlsResultAdd(1, 0, nil) c.tlsResultAdd(1, 0, nil)
@ -1171,7 +1173,7 @@ func (c *Client) Close() (rerr error) {
c.xwriteline("QUIT") c.xwriteline("QUIT")
if err := c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { if err := c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
c.log.Infox("setting read deadline for reading quit response", err) c.log.Infox("setting read deadline for reading quit response", err)
} else if _, err := bufs.Readline(c.r); err != nil { } else if _, err := bufs.Readline(c.log, c.r); err != nil {
rerr = fmt.Errorf("reading response to quit command: %v", err) rerr = fmt.Errorf("reading response to quit command: %v", err)
c.log.Debugx("reading quit response", err) c.log.Debugx("reading quit response", err)
} }

36
smtpclient/client_test.go
View file

@ -21,6 +21,8 @@ import (
"testing" "testing"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/sasl" "github.com/mjl-/mox/sasl"
@ -33,9 +35,9 @@ var localhost = dns.Domain{ASCII: "localhost"}
func TestClient(t *testing.T) { func TestClient(t *testing.T) {
ctx := context.Background() ctx := context.Background()
log := mlog.New("smtpclient") log := mlog.New("smtpclient", nil)
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelTrace}) mlog.SetConfig(map[string]slog.Level{"": mlog.LevelTrace})
type options struct { type options struct {
pipelining bool pipelining bool
@ -281,7 +283,7 @@ func TestClient(t *testing.T) {
result <- err result <- err
panic("stop") panic("stop")
} }
c, err := New(ctx, log, clientConn, opts.tlsMode, opts.tlsPKIX, localhost, opts.tlsHostname, Opts{Auth: auths, RootCAs: opts.roots}) c, err := New(ctx, log.Logger, clientConn, opts.tlsMode, opts.tlsPKIX, localhost, opts.tlsHostname, Opts{Auth: auths, RootCAs: opts.roots})
if (err == nil) != (expClientErr == nil) || err != nil && !errors.As(err, reflect.New(reflect.ValueOf(expClientErr).Type()).Interface()) && !errors.Is(err, expClientErr) { if (err == nil) != (expClientErr == nil) || err != nil && !errors.As(err, reflect.New(reflect.ValueOf(expClientErr).Type()).Interface()) && !errors.Is(err, expClientErr) {
fail("new client: got err %v, expected %#v", err, expClientErr) fail("new client: got err %v, expected %#v", err, expClientErr)
} }
@ -382,13 +384,13 @@ test
func TestErrors(t *testing.T) { func TestErrors(t *testing.T) {
ctx := context.Background() ctx := context.Background()
log := mlog.New("") log := mlog.New("smtpclient", nil)
// Invalid greeting. // Invalid greeting.
run(t, func(s xserver) { run(t, func(s xserver) {
s.writeline("bogus") // Invalid, should be "220 <hostname>". s.writeline("bogus") // Invalid, should be "220 <hostname>".
}, func(conn net.Conn) { }, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error var xerr Error
if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent { if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err)) panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err))
@ -399,7 +401,7 @@ func TestErrors(t *testing.T) {
run(t, func(s xserver) { run(t, func(s xserver) {
s.conn.Close() s.conn.Close()
}, func(conn net.Conn) { }, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error var xerr Error
if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) || !errors.As(err, &xerr) || xerr.Permanent { if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v (%v), expected ErrUnexpectedEOF without Permanent", err, err)) panic(fmt.Errorf("got %#v (%v), expected ErrUnexpectedEOF without Permanent", err, err))
@ -410,7 +412,7 @@ func TestErrors(t *testing.T) {
run(t, func(s xserver) { run(t, func(s xserver) {
s.writeline("521 not accepting connections") s.writeline("521 not accepting connections")
}, func(conn net.Conn) { }, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error var xerr Error
if err == nil || !errors.Is(err, ErrStatus) || !errors.As(err, &xerr) || !xerr.Permanent { if err == nil || !errors.Is(err, ErrStatus) || !errors.As(err, &xerr) || !xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrStatus with Permanent", err)) panic(fmt.Errorf("got %#v, expected ErrStatus with Permanent", err))
@ -421,7 +423,7 @@ func TestErrors(t *testing.T) {
run(t, func(s xserver) { run(t, func(s xserver) {
s.writeline("2200 mox.example") // Invalid, too many digits. s.writeline("2200 mox.example") // Invalid, too many digits.
}, func(conn net.Conn) { }, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error var xerr Error
if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent { if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err)) panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err))
@ -435,7 +437,7 @@ func TestErrors(t *testing.T) {
s.writeline("250-mox.example") s.writeline("250-mox.example")
s.writeline("500 different code") // Invalid. s.writeline("500 different code") // Invalid.
}, func(conn net.Conn) { }, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error var xerr Error
if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent { if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err)) panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err))
@ -451,7 +453,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:") s.readline("MAIL FROM:")
s.writeline("550 5.7.0 not allowed") s.writeline("550 5.7.0 not allowed")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -471,7 +473,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:") s.readline("MAIL FROM:")
s.writeline("451 bad sender") s.writeline("451 bad sender")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -493,7 +495,7 @@ func TestErrors(t *testing.T) {
s.readline("RCPT TO:") s.readline("RCPT TO:")
s.writeline("451") s.writeline("451")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -517,7 +519,7 @@ func TestErrors(t *testing.T) {
s.readline("DATA") s.readline("DATA")
s.writeline("550 no!") s.writeline("550 no!")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -537,7 +539,7 @@ func TestErrors(t *testing.T) {
s.readline("STARTTLS") s.readline("STARTTLS")
s.writeline("502 command not implemented") s.writeline("502 command not implemented")
}, func(conn net.Conn) { }, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSRequiredStartTLS, true, localhost, dns.Domain{ASCII: "mox.example"}, Opts{}) _, err := New(ctx, log.Logger, conn, TLSRequiredStartTLS, true, localhost, dns.Domain{ASCII: "mox.example"}, Opts{})
var xerr Error var xerr Error
if err == nil || !errors.Is(err, ErrTLS) || !errors.As(err, &xerr) || !xerr.Permanent { if err == nil || !errors.Is(err, ErrTLS) || !errors.As(err, &xerr) || !xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrTLS with Permanent", err)) panic(fmt.Errorf("got %#v, expected ErrTLS with Permanent", err))
@ -553,7 +555,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:") s.readline("MAIL FROM:")
s.writeline("451 enough") s.writeline("451 enough")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSSkip, false, localhost, dns.Domain{ASCII: "mox.example"}, Opts{}) c, err := New(ctx, log.Logger, conn, TLSSkip, false, localhost, dns.Domain{ASCII: "mox.example"}, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -583,7 +585,7 @@ func TestErrors(t *testing.T) {
s.readline("DATA") s.readline("DATA")
s.writeline("550 not now") s.writeline("550 not now")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -613,7 +615,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:") s.readline("MAIL FROM:")
s.writeline("550 ok") s.writeline("550 ok")
}, func(conn net.Conn) { }, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil { if err != nil {
panic(err) panic(err)
} }

11
smtpclient/dial.go
View file

@ -6,6 +6,8 @@ import (
"net" "net"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
@ -51,7 +53,8 @@ type Dialer interface {
// If we have fully specified local SMTP listener IPs, we set those for the // If we have fully specified local SMTP listener IPs, we set those for the
// outgoing connection. The admin probably configured these same IPs in SPF, but // outgoing connection. The admin probably configured these same IPs in SPF, but
// others possibly not. // others possibly not.
func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) { func Dial(ctx context.Context, elog *slog.Logger, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) {
log := mlog.New("smtpclient", elog)
timeout := 30 * time.Second timeout := 30 * time.Second
if deadline, ok := ctx.Deadline(); ok && len(ips) > 0 { if deadline, ok := ctx.Deadline(); ok && len(ips) > 0 {
timeout = time.Until(deadline) / time.Duration(len(ips)) timeout = time.Until(deadline) / time.Duration(len(ips))
@ -61,7 +64,7 @@ func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain,
var lastIP net.IP var lastIP net.IP
for _, ip := range ips { for _, ip := range ips {
addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port)) addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port))
log.Debug("dialing host", mlog.Field("addr", addr)) log.Debug("dialing host", slog.String("addr", addr))
var laddr net.Addr var laddr net.Addr
for _, lip := range mox.Conf.Static.SpecifiedSMTPListenIPs { for _, lip := range mox.Conf.Static.SpecifiedSMTPListenIPs {
ipIs4 := ip.To4() != nil ipIs4 := ip.To4() != nil
@ -73,12 +76,12 @@ func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain,
} }
conn, err := dial(ctx, dialer, timeout, addr, laddr) conn, err := dial(ctx, dialer, timeout, addr, laddr)
if err == nil { if err == nil {
log.Debug("connected to host", mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr)) log.Debug("connected to host", slog.Any("host", host), slog.String("addr", addr), slog.Any("laddr", laddr))
name := host.String() name := host.String()
dialedIPs[name] = append(dialedIPs[name], ip) dialedIPs[name] = append(dialedIPs[name], ip)
return conn, ip, nil return conn, ip, nil
} }
log.Debugx("connection attempt", err, mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr)) log.Debugx("connection attempt", err, slog.Any("host", host), slog.String("addr", addr), slog.Any("laddr", laddr))
lastErr = err lastErr = err
lastIP = ip lastIP = ip
} }

10
smtpclient/dial_test.go
View file

@ -14,7 +14,7 @@ import (
func TestDialHost(t *testing.T) { func TestDialHost(t *testing.T) {
// We mostly want to test that dialing a second time switches to the other address family. // We mostly want to test that dialing a second time switches to the other address family.
ctxbg := context.Background() ctxbg := context.Background()
log := mlog.New("smtpclient") log := mlog.New("smtpclient", nil)
resolver := dns.MockResolver{ resolver := dns.MockResolver{
A: map[string][]string{ A: map[string][]string{
@ -37,20 +37,20 @@ func TestDialHost(t *testing.T) {
} }
dialedIPs := map[string][]net.IP{} dialedIPs := map[string][]net.IP{}
_, _, _, ips, dualstack, err := GatherIPs(ctxbg, log, resolver, ipdomain("dualstack.example"), dialedIPs) _, _, _, ips, dualstack, err := GatherIPs(ctxbg, log.Logger, resolver, ipdomain("dualstack.example"), dialedIPs)
if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}) || !dualstack { if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}) || !dualstack {
t.Fatalf("expected err nil, address 10.0.0.1,2001:db8::1, dualstack true, got %v %v %v", err, ips, dualstack) t.Fatalf("expected err nil, address 10.0.0.1,2001:db8::1, dualstack true, got %v %v %v", err, ips, dualstack)
} }
_, ip, err := Dial(ctxbg, log, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) _, ip, err := Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs)
if err != nil || ip.String() != "10.0.0.1" { if err != nil || ip.String() != "10.0.0.1" {
t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack) t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack)
} }
_, _, _, ips, dualstack, err = GatherIPs(ctxbg, log, resolver, ipdomain("dualstack.example"), dialedIPs) _, _, _, ips, dualstack, err = GatherIPs(ctxbg, log.Logger, resolver, ipdomain("dualstack.example"), dialedIPs)
if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("10.0.0.1")}) || !dualstack { if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("10.0.0.1")}) || !dualstack {
t.Fatalf("expected err nil, address 2001:db8::1,10.0.0.1, dualstack true, got %v %v %v", err, ips, dualstack) t.Fatalf("expected err nil, address 2001:db8::1,10.0.0.1, dualstack true, got %v %v %v", err, ips, dualstack)
} }
_, ip, err = Dial(ctxbg, log, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) _, ip, err = Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs)
if err != nil || ip.String() != "2001:db8::1" { if err != nil || ip.String() != "2001:db8::1" {
t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack) t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack)
} }

36
smtpclient/gather.go
View file

@ -12,6 +12,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
@ -45,9 +47,11 @@ var (
// were found, both the original and expanded next-hops must be authentic for DANE // were found, both the original and expanded next-hops must be authentic for DANE
// to apply. For a non-IP with no MX records found, the authentic result can be // to apply. For a non-IP with no MX records found, the authentic result can be
// used to decide which of the names to use as TLSA base domain. // used to decide which of the names to use as TLSA base domain.
func GatherDestinations(ctx context.Context, log *mlog.Log, resolver dns.Resolver, origNextHop dns.IPDomain) (haveMX, origNextHopAuthentic, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, hosts []dns.IPDomain, permanent bool, err error) { func GatherDestinations(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, origNextHop dns.IPDomain) (haveMX, origNextHopAuthentic, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, hosts []dns.IPDomain, permanent bool, err error) {
// ../rfc/5321:3824 // ../rfc/5321:3824
log := mlog.New("smtpclient", elog)
// IP addresses are dialed directly, and don't have TLSA records. // IP addresses are dialed directly, and don't have TLSA records.
if len(origNextHop.IP) > 0 { if len(origNextHop.IP) > 0 {
return false, false, false, expandedNextHop, []dns.IPDomain{origNextHop}, false, nil return false, false, false, expandedNextHop, []dns.IPDomain{origNextHop}, false, nil
@ -167,7 +171,9 @@ func GatherDestinations(ctx context.Context, log *mlog.Log, resolver dns.Resolve
// GatherIPs looks up the IPs to try for connecting to host, with the IPs ordered // GatherIPs looks up the IPs to try for connecting to host, with the IPs ordered
// to take previous attempts into account. For use with DANE, the CNAME-expanded // to take previous attempts into account. For use with DANE, the CNAME-expanded
// name is returned, and whether the DNS responses were authentic. // name is returned, and whether the DNS responses were authentic.
func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host dns.IPDomain, dialedIPs map[string][]net.IP) (authentic bool, expandedAuthentic bool, expandedHost dns.Domain, ips []net.IP, dualstack bool, rerr error) { func GatherIPs(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, host dns.IPDomain, dialedIPs map[string][]net.IP) (authentic bool, expandedAuthentic bool, expandedHost dns.Domain, ips []net.IP, dualstack bool, rerr error) {
log := mlog.New("smtpclient", elog)
if len(host.IP) > 0 { if len(host.IP) > 0 {
return false, false, dns.Domain{}, []net.IP{host.IP}, false, nil return false, false, dns.Domain{}, []net.IP{host.IP}, false, nil
} }
@ -250,7 +256,7 @@ func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host d
// Prefer "i" if it is the same as last and we should be preferring it. // Prefer "i" if it is the same as last and we should be preferring it.
return preferPrev && ips[i].Equal(prevIP) return preferPrev && ips[i].Equal(prevIP)
}) })
log.Debug("ordered ips for dialing", mlog.Field("ips", ips)) log.Debug("ordered ips for dialing", slog.Any("ips", ips))
} }
return return
} }
@ -268,7 +274,9 @@ func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host d
// must do TLS, but not verify the remote TLS certificate. // must do TLS, but not verify the remote TLS certificate.
// //
// Returned values are always meaningful, also when an error was returned. // Returned values are always meaningful, also when an error was returned.
func GatherTLSA(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain) (daneRequired bool, daneRecords []adns.TLSA, tlsaBaseDomain dns.Domain, err error) { func GatherTLSA(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain) (daneRequired bool, daneRecords []adns.TLSA, tlsaBaseDomain dns.Domain, err error) {
log := mlog.New("smtpclient", elog)
// ../rfc/7672:912 // ../rfc/7672:912
// This function is only called when the lookup of host was authentic. // This function is only called when the lookup of host was authentic.
@ -288,32 +296,32 @@ func GatherTLSA(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host
} }
if len(l) == 0 || err != nil { if len(l) == 0 || err != nil {
daneRequired = err != nil daneRequired = err != nil
log.Debugx("gathering tlsa records failed", err, mlog.Field("danerequired", daneRequired), mlog.Field("basedomain", tlsaBaseDomain)) log.Debugx("gathering tlsa records failed", err, slog.Bool("danerequired", daneRequired), slog.Any("basedomain", tlsaBaseDomain))
return daneRequired, nil, tlsaBaseDomain, err return daneRequired, nil, tlsaBaseDomain, err
} }
daneRequired = len(l) > 0 daneRequired = len(l) > 0
l = filterUsableTLSARecords(log, l) l = filterUsableTLSARecords(log, l)
log.Debug("tlsa records exist", mlog.Field("danerequired", daneRequired), mlog.Field("records", l), mlog.Field("basedomain", tlsaBaseDomain)) log.Debug("tlsa records exist", slog.Bool("danerequired", daneRequired), slog.Any("records", l), slog.Any("basedomain", tlsaBaseDomain))
return daneRequired, l, tlsaBaseDomain, err return daneRequired, l, tlsaBaseDomain, err
} }
// lookupTLSACNAME composes a TLSA domain name to lookup, follows CNAMEs and looks // lookupTLSACNAME composes a TLSA domain name to lookup, follows CNAMEs and looks
// up TLSA records. no TLSA records exist, a nil error is returned as it means // up TLSA records. no TLSA records exist, a nil error is returned as it means
// the host does not opt-in to DANE. // the host does not opt-in to DANE.
func lookupTLSACNAME(ctx context.Context, log *mlog.Log, resolver dns.Resolver, port int, protocol string, host dns.Domain) (l []adns.TLSA, rerr error) { func lookupTLSACNAME(ctx context.Context, log mlog.Log, resolver dns.Resolver, port int, protocol string, host dns.Domain) (l []adns.TLSA, rerr error) {
name := fmt.Sprintf("_%d._%s.%s", port, protocol, host.ASCII+".") name := fmt.Sprintf("_%d._%s.%s", port, protocol, host.ASCII+".")
for i := 0; ; i++ { for i := 0; ; i++ {
cname, result, err := resolver.LookupCNAME(ctx, name) cname, result, err := resolver.LookupCNAME(ctx, name)
if dns.IsNotFound(err) { if dns.IsNotFound(err) {
if !result.Authentic { if !result.Authentic {
log.Debugx("cname nxdomain result during tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name)) log.Debugx("cname nxdomain result during tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name))
return nil, nil return nil, nil
} }
break break
} else if err != nil { } else if err != nil {
return nil, fmt.Errorf("looking up cname for tlsa candidate base domain: %w", err) return nil, fmt.Errorf("looking up cname for tlsa candidate base domain: %w", err)
} else if !result.Authentic { } else if !result.Authentic {
log.Debugx("cname result during tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name)) log.Debugx("cname result during tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name))
return nil, nil return nil, nil
} }
if i == 10 { if i == 10 {
@ -325,18 +333,18 @@ func lookupTLSACNAME(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
var err error var err error
l, result, err = resolver.LookupTLSA(ctx, 0, "", name) l, result, err = resolver.LookupTLSA(ctx, 0, "", name)
if dns.IsNotFound(err) || err == nil && len(l) == 0 { if dns.IsNotFound(err) || err == nil && len(l) == 0 {
log.Debugx("no tlsa records for host, not doing dane", err, mlog.Field("host", host), mlog.Field("name", name), mlog.Field("authentic", result.Authentic)) log.Debugx("no tlsa records for host, not doing dane", err, slog.Any("host", host), slog.String("name", name), slog.Bool("authentic", result.Authentic))
return nil, nil return nil, nil
} else if err != nil { } else if err != nil {
return nil, fmt.Errorf("looking up tlsa records for tlsa candidate base domain: %w", err) return nil, fmt.Errorf("looking up tlsa records for tlsa candidate base domain: %w", err)
} else if !result.Authentic { } else if !result.Authentic {
log.Debugx("tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name)) log.Debugx("tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name))
return nil, nil return nil, nil
} }
return l, nil return l, nil
} }
func filterUsableTLSARecords(log *mlog.Log, l []adns.TLSA) []adns.TLSA { func filterUsableTLSARecords(log mlog.Log, l []adns.TLSA) []adns.TLSA {
// Gather "usable" records. ../rfc/7672:708 // Gather "usable" records. ../rfc/7672:708
o := 0 o := 0
for _, r := range l { for _, r := range l {
@ -368,12 +376,12 @@ func filterUsableTLSARecords(log *mlog.Log, l []adns.TLSA) []adns.TLSA {
} }
case adns.TLSAMatchTypeSHA256: case adns.TLSAMatchTypeSHA256:
if len(r.CertAssoc) != sha256.Size { if len(r.CertAssoc) != sha256.Size {
log.Debug("dane tlsa record with wrong data size for sha2-256", mlog.Field("got", len(r.CertAssoc)), mlog.Field("expect", sha256.Size)) log.Debug("dane tlsa record with wrong data size for sha2-256", slog.Int("got", len(r.CertAssoc)), slog.Int("expect", sha256.Size))
continue continue
} }
case adns.TLSAMatchTypeSHA512: case adns.TLSAMatchTypeSHA512:
if len(r.CertAssoc) != sha512.Size { if len(r.CertAssoc) != sha512.Size {
log.Debug("dane tlsa record with wrong data size for sha2-512", mlog.Field("got", len(r.CertAssoc)), mlog.Field("expect", sha512.Size)) log.Debug("dane tlsa record with wrong data size for sha2-512", slog.Int("got", len(r.CertAssoc)), slog.Int("expect", sha512.Size))
continue continue
} }
default: default:

12
smtpclient/gather_test.go
View file

@ -47,7 +47,7 @@ func ipdomains(s ...string) (l []dns.IPDomain) {
// exist or has temporary error. // exist or has temporary error.
func TestGatherDestinations(t *testing.T) { func TestGatherDestinations(t *testing.T) {
ctxbg := context.Background() ctxbg := context.Background()
log := mlog.New("smtpclient") log := mlog.New("smtpclient", nil)
resolver := dns.MockResolver{ resolver := dns.MockResolver{
MX: map[string][]*net.MX{ MX: map[string][]*net.MX{
@ -89,7 +89,7 @@ func TestGatherDestinations(t *testing.T) {
test := func(ipd dns.IPDomain, expHosts []dns.IPDomain, expDomain dns.Domain, expPerm, expAuthic, expExpAuthic bool, expErr error) { test := func(ipd dns.IPDomain, expHosts []dns.IPDomain, expDomain dns.Domain, expPerm, expAuthic, expExpAuthic bool, expErr error) {
t.Helper() t.Helper()
_, authic, authicExp, ed, hosts, perm, err := GatherDestinations(ctxbg, log, resolver, ipd) _, authic, authicExp, ed, hosts, perm, err := GatherDestinations(ctxbg, log.Logger, resolver, ipd)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
// todo: could also check the individual errors? code currently does not have structured errors. // todo: could also check the individual errors? code currently does not have structured errors.
t.Fatalf("gather hosts: %v, expected %v", err, expErr) t.Fatalf("gather hosts: %v, expected %v", err, expErr)
@ -134,7 +134,7 @@ func TestGatherDestinations(t *testing.T) {
func TestGatherIPs(t *testing.T) { func TestGatherIPs(t *testing.T) {
ctxbg := context.Background() ctxbg := context.Background()
log := mlog.New("smtpclient") log := mlog.New("smtpclient", nil)
resolver := dns.MockResolver{ resolver := dns.MockResolver{
A: map[string][]string{ A: map[string][]string{
@ -164,7 +164,7 @@ func TestGatherIPs(t *testing.T) {
test := func(host dns.IPDomain, expAuthic, expAuthicExp bool, expHostExp dns.Domain, expIPs []net.IP, expErr any) { test := func(host dns.IPDomain, expAuthic, expAuthicExp bool, expHostExp dns.Domain, expIPs []net.IP, expErr any) {
t.Helper() t.Helper()
authic, authicExp, hostExp, ips, _, err := GatherIPs(ctxbg, log, resolver, host, nil) authic, authicExp, hostExp, ips, _, err := GatherIPs(ctxbg, log.Logger, resolver, host, nil)
if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) { if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) {
// todo: could also check the individual errors? // todo: could also check the individual errors?
t.Fatalf("gather hosts: %v, expected %v", err, expErr) t.Fatalf("gather hosts: %v, expected %v", err, expErr)
@ -207,7 +207,7 @@ func TestGatherIPs(t *testing.T) {
func TestGatherTLSA(t *testing.T) { func TestGatherTLSA(t *testing.T) {
ctxbg := context.Background() ctxbg := context.Background()
log := mlog.New("smtpclient") log := mlog.New("smtpclient", nil)
record := func(usage, selector, matchType uint8) adns.TLSA { record := func(usage, selector, matchType uint8) adns.TLSA {
return adns.TLSA{ return adns.TLSA{
@ -253,7 +253,7 @@ func TestGatherTLSA(t *testing.T) {
test := func(host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain, expDANERequired bool, expRecords []adns.TLSA, expBaseDom dns.Domain, expErr any) { test := func(host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain, expDANERequired bool, expRecords []adns.TLSA, expBaseDom dns.Domain, expErr any) {
t.Helper() t.Helper()
daneReq, records, baseDom, err := GatherTLSA(ctxbg, log, resolver, host, expandedAuthentic, expandedHost) daneReq, records, baseDom, err := GatherTLSA(ctxbg, log.Logger, resolver, host, expandedAuthentic, expandedHost)
if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) { if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) {
// todo: could also check the individual errors? // todo: could also check the individual errors?
t.Fatalf("gather tlsa: %v, expected %v", err, expErr) t.Fatalf("gather tlsa: %v, expected %v", err, expErr)

9
smtpserver/alignment.go
View file

@ -5,6 +5,7 @@ import (
"github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/spf" "github.com/mjl-/mox/spf"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
@ -12,9 +13,9 @@ import (
// Alignment compares the msgFromDomain with the dkim and spf results, and returns // Alignment compares the msgFromDomain with the dkim and spf results, and returns
// a validation, one of: Strict, Relaxed, None. // a validation, one of: Strict, Relaxed, None.
func alignment(ctx context.Context, msgFromDomain dns.Domain, dkimResults []dkim.Result, spfStatus spf.Status, spfIdentity *dns.Domain) store.Validation { func alignment(ctx context.Context, log mlog.Log, msgFromDomain dns.Domain, dkimResults []dkim.Result, spfStatus spf.Status, spfIdentity *dns.Domain) store.Validation {
var strict, relaxed bool var strict, relaxed bool
msgFromOrgDomain := publicsuffix.Lookup(ctx, msgFromDomain) msgFromOrgDomain := publicsuffix.Lookup(ctx, log.Logger, msgFromDomain)
// todo: should take temperror and permerror into account. // todo: should take temperror and permerror into account.
for _, dr := range dkimResults { for _, dr := range dkimResults {
@ -25,12 +26,12 @@ func alignment(ctx context.Context, msgFromDomain dns.Domain, dkimResults []dkim
strict = true strict = true
break break
} else { } else {
relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, dr.Sig.Domain) relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, log.Logger, dr.Sig.Domain)
} }
} }
if !strict && spfStatus == spf.StatusPass { if !strict && spfStatus == spf.StatusPass {
strict = msgFromDomain == *spfIdentity strict = msgFromDomain == *spfIdentity
relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, *spfIdentity) relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, log.Logger, *spfIdentity)
} }
if strict { if strict {
return store.ValidationStrict return store.ValidationStrict

46
smtpserver/analyze.go
View file

@ -8,6 +8,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dkim"
@ -89,7 +91,7 @@ func isListDomain(d delivery, ld dns.Domain) bool {
return false return false
} }
func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis { func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis {
var headers string var headers string
mailbox := d.rcptAcc.destination.Mailbox mailbox := d.rcptAcc.destination.Mailbox
@ -158,7 +160,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
d.m.MailboxID = mb.ID d.m.MailboxID = mb.ID
d.m.MailboxDestinedID = mb.ID d.m.MailboxDestinedID = mb.ID
} else { } else {
log.Debug("mailbox not found in database", mlog.Field("mailbox", mailbox)) log.Debug("mailbox not found in database", slog.String("mailbox", mailbox))
} }
return nil return nil
} }
@ -206,17 +208,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
if d.dmarcResult.Status != dmarc.StatusPass { if d.dmarcResult.Status != dmarc.StatusPass {
log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report") log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")
headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n" headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"
} else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { } else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
log.Infox("parsing dmarc aggregate report", err) log.Infox("parsing dmarc aggregate report", err)
headers += "X-Mox-DMARCReport-Error: could not parse report\r\n" headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"
} else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil { } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {
log.Infox("parsing domain in dmarc aggregate report", err) log.Infox("parsing domain in dmarc aggregate report", err)
headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n" headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"
} else if _, ok := mox.Conf.Domain(d); !ok { } else if _, ok := mox.Conf.Domain(d); !ok {
log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d)) log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d))
headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n" headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"
} else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 { } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {
log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0))) log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n" headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"
} else { } else {
dmarcReport = report dmarcReport = report
@ -230,7 +232,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
matchesDomain := func(sigDomain dns.Domain) bool { matchesDomain := func(sigDomain dns.Domain) bool {
// RFC seems to require exact DKIM domain match with submitt and message From, we // RFC seems to require exact DKIM domain match with submitt and message From, we
// also allow msgFrom to be subdomain. ../rfc/8460:322 // also allow msgFrom to be subdomain. ../rfc/8460:322
return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, sigDomain) return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain)
} }
// Valid DKIM signature for domain must be present. We take "valid" to assume // Valid DKIM signature for domain must be present. We take "valid" to assume
// "passing", not "syntactically valid". We also check for "tlsrpt" as service. // "passing", not "syntactically valid". We also check for "tlsrpt" as service.
@ -255,13 +257,13 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
if !ok { if !ok {
log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report") log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")
headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n" headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"
} else if report, err := tlsrpt.ParseMessage(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { } else if report, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
log.Infox("parsing tls report", err) log.Infox("parsing tls report", err)
headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n" headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"
} else { } else {
var known bool var known bool
for _, p := range report.Policies { for _, p := range report.Policies {
log.Info("tlsrpt policy domain", mlog.Field("domain", p.Policy.Domain)) log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain))
if d, err := dns.ParseDomain(p.Policy.Domain); err != nil { if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {
log.Infox("parsing domain in tls report", err) log.Infox("parsing domain in tls report", err)
} else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain { } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {
@ -297,10 +299,10 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
}) })
}) })
if err != nil { if err != nil {
log.Infox("determining reputation", err, mlog.Field("message", d.m)) log.Infox("determining reputation", err, slog.Any("message", d.m))
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError) return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)
} }
log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method))) log.Info("reputation analyzed", slog.Bool("conclusive", conclusive), slog.Any("isjunk", isjunk), slog.String("method", string(method)))
if conclusive { if conclusive {
if !*isjunk { if !*isjunk {
return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers} return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers}
@ -340,9 +342,9 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
log.Errorx("get key for verifying subject token", err) log.Errorx("get key for verifying subject token", err)
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError) return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)
} }
err = subjectpass.Verify(log, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period) err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)
pass := err == nil pass := err == nil
log.Infox("pass by subject token", err, mlog.Field("pass", pass)) log.Infox("pass by subject token", err, slog.Bool("pass", pass))
if pass { if pass {
return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers} return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers}
} }
@ -395,11 +397,11 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
reason = reasonJunkContent reason = reasonJunkContent
if suspiciousIPrevFail && threshold > 0.25 { if suspiciousIPrevFail && threshold > 0.25 {
threshold = 0.25 threshold = 0.25
log.Info("setting junk threshold due to iprev fail", mlog.Field("threshold", threshold)) log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold))
reason = reasonJunkContentStrict reason = reasonJunkContentStrict
} else if !d.tls && threshold > 0.25 { } else if !d.tls && threshold > 0.25 {
threshold = 0.25 threshold = 0.25
log.Info("setting junk threshold due to plaintext smtp", mlog.Field("threshold", threshold)) log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold))
reason = reasonJunkContentStrict reason = reasonJunkContentStrict
} else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) { } else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) {
// A common theme in junk messages is your recipient address not being in the To/Cc // A common theme in junk messages is your recipient address not being in the To/Cc
@ -407,12 +409,12 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
// providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be // providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be
// sent with matching Bcc headers. We don't get here for known senders. // sent with matching Bcc headers. We don't get here for known senders.
threshold = 0.25 threshold = 0.25
log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", mlog.Field("threshold", threshold)) log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold))
reason = reasonJunkContentStrict reason = reasonJunkContentStrict
} }
accept = contentProb <= threshold accept = contentProb <= threshold
junkSubjectpass = contentProb < threshold-0.2 junkSubjectpass = contentProb < threshold-0.2
log.Info("content analyzed", mlog.Field("accept", accept), mlog.Field("contentprob", contentProb), mlog.Field("subjectpass", junkSubjectpass)) log.Info("content analyzed", slog.Bool("accept", accept), slog.Float64("contentprob", contentProb), slog.Bool("subjectpass", junkSubjectpass))
} else if err != store.ErrNoJunkFilter { } else if err != store.ErrNoJunkFilter {
log.Errorx("open junkfilter", err) log.Errorx("open junkfilter", err)
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError) return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)
@ -426,18 +428,18 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
blocked := func(zone dns.Domain) bool { blocked := func(zone dns.Domain) bool {
dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second) dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)
defer dnsblcancel() defer dnsblcancel()
if !checkDNSBLHealth(dnsblctx, resolver, zone) { if !checkDNSBLHealth(dnsblctx, log, resolver, zone) {
log.Info("dnsbl not healthy, skipping", mlog.Field("zone", zone)) log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone))
return false return false
} }
status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(d.m.RemoteIP)) status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP))
dnsblcancel() dnsblcancel()
if status == dnsbl.StatusFail { if status == dnsbl.StatusFail {
log.Info("rejecting due to listing in dnsbl", mlog.Field("zone", zone), mlog.Field("explanation", expl)) log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl))
return true return true
} else if err != nil { } else if err != nil {
log.Infox("dnsbl lookup", err, mlog.Field("zone", zone), mlog.Field("status", status)) log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status))
} }
return false return false
} }
@ -459,7 +461,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) { if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {
log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation") log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")
pass := subjectpass.Generate(d.msgFrom, []byte(subjectpassKey), time.Now()) pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now())
return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass) return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)
} }

5
smtpserver/dnsbl.go
View file

@ -8,6 +8,7 @@ import (
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dnsbl" "github.com/mjl-/mox/dnsbl"
"github.com/mjl-/mox/mlog"
) )
var dnsblHealth = struct { var dnsblHealth = struct {
@ -23,12 +24,12 @@ type dnsblStatus struct {
} }
// checkDNSBLHealth checks healthiness of DNSBL "zone", keeping the result cached for 4 hours. // checkDNSBLHealth checks healthiness of DNSBL "zone", keeping the result cached for 4 hours.
func checkDNSBLHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rok bool) { func checkDNSBLHealth(ctx context.Context, log mlog.Log, resolver dns.Resolver, zone dns.Domain) (rok bool) {
dnsblHealth.Lock() dnsblHealth.Lock()
defer dnsblHealth.Unlock() defer dnsblHealth.Unlock()
status, ok := dnsblHealth.zones[zone] status, ok := dnsblHealth.zones[zone]
if !ok || time.Since(status.last) > 4*time.Hour { if !ok || time.Since(status.last) > 4*time.Hour {
status.err = dnsbl.CheckHealth(ctx, resolver, zone) status.err = dnsbl.CheckHealth(ctx, log.Logger, resolver, zone)
status.last = time.Now() status.last = time.Now()
dnsblHealth.zones[zone] = status dnsblHealth.zones[zone] = status
} }

2
smtpserver/dsn.go
View file

@ -24,7 +24,7 @@ func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, req
} }
} }
f, err := store.CreateMessageTemp("smtp-dsn") f, err := store.CreateMessageTemp(c.log, "smtp-dsn")
if err != nil { if err != nil {
return fmt.Errorf("creating temp file: %w", err) return fmt.Errorf("creating temp file: %w", err)
} }

6
smtpserver/fuzz_test.go
View file

@ -9,6 +9,7 @@ import (
"time" "time"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/queue" "github.com/mjl-/mox/queue"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
@ -30,17 +31,18 @@ func FuzzServer(f *testing.F) {
f.Add("NOOP") f.Add("NOOP")
f.Add("QUIT") f.Add("QUIT")
log := mlog.New("smtpserver", nil)
mox.Context = ctxbg mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/smtpserverfuzz/mox.conf") mox.ConfigStaticPath = filepath.FromSlash("../testdata/smtpserverfuzz/mox.conf")
mox.MustLoadConfig(true, false) mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
os.RemoveAll(dataDir) os.RemoveAll(dataDir)
acc, err := store.OpenAccount("mjl") acc, err := store.OpenAccount(log, "mjl")
if err != nil { if err != nil {
f.Fatalf("open account: %v", err) f.Fatalf("open account: %v", err)
} }
defer acc.Close() defer acc.Close()
err = acc.SetPassword("testtest") err = acc.SetPassword(log, "testtest")
if err != nil { if err != nil {
f.Fatalf("set password: %v", err) f.Fatalf("set password: %v", err)
} }

8
smtpserver/rejects.go
View file

@ -8,6 +8,8 @@ import (
"io" "io"
"os" "os"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
@ -17,15 +19,15 @@ import (
) )
// rejectPresent returns whether the message is already present in the rejects mailbox. // rejectPresent returns whether the message is already present in the rejects mailbox.
func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *store.Message, f *os.File) (present bool, msgID string, hash []byte, rerr error) { func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *store.Message, f *os.File) (present bool, msgID string, hash []byte, rerr error) {
if p, err := message.Parse(log, false, store.FileMsgReader(m.MsgPrefix, f)); err != nil { if p, err := message.Parse(log.Logger, false, store.FileMsgReader(m.MsgPrefix, f)); err != nil {
log.Infox("parsing reject message for message-id", err) log.Infox("parsing reject message for message-id", err)
} else if header, err := p.Header(); err != nil { } else if header, err := p.Header(); err != nil {
log.Infox("parsing reject message header for message-id", err) log.Infox("parsing reject message header for message-id", err)
} else { } else {
msgID, _, err = message.MessageIDCanonical(header.Get("Message-Id")) msgID, _, err = message.MessageIDCanonical(header.Get("Message-Id"))
if err != nil { if err != nil {
log.Debugx("parsing message-id for reject", err, mlog.Field("messageid", header.Get("Message-Id"))) log.Debugx("parsing message-id for reject", err, slog.String("messageid", header.Get("Message-Id")))
} }
} }

6
smtpserver/reputation.go
View file

@ -5,6 +5,8 @@ import (
"fmt" "fmt"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
@ -96,7 +98,7 @@ const (
// ../rfc/6376:1915 // ../rfc/6376:1915
// ../rfc/6376:3716 // ../rfc/6376:3716
// ../rfc/7208:2167 // ../rfc/7208:2167
func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rconclusive bool, rmethod reputationMethod, rerr error) { func reputation(tx *bstore.Tx, log mlog.Log, m *store.Message) (rjunk *bool, rconclusive bool, rmethod reputationMethod, rerr error) {
boolptr := func(v bool) *bool { boolptr := func(v bool) *bool {
return &v return &v
} }
@ -141,7 +143,7 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc
xmessageList := func(q *bstore.Query[store.Message], descr string) []store.Message { xmessageList := func(q *bstore.Query[store.Message], descr string) []store.Message {
t0 := time.Now() t0 := time.Now()
l, err := q.List() l, err := q.List()
log.Debugx("querying messages for reputation", err, mlog.Field("msgs", len(l)), mlog.Field("descr", descr), mlog.Field("queryduration", time.Since(t0))) log.Debugx("querying messages for reputation", err, slog.Int("msgs", len(l)), slog.String("descr", descr), slog.Duration("queryduration", time.Since(t0)))
if err != nil { if err != nil {
panic(queryError(fmt.Sprintf("listing messages: %v", err))) panic(queryError(fmt.Sprintf("listing messages: %v", err)))
} }

11
smtpserver/reputation_test.go
View file

@ -11,11 +11,14 @@ import (
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
var pkglog = mlog.New("smtpserver", nil)
func TestReputation(t *testing.T) { func TestReputation(t *testing.T) {
boolptr := func(v bool) *bool { boolptr := func(v bool) *bool {
return &v return &v
@ -26,6 +29,8 @@ func TestReputation(t *testing.T) {
now := time.Now() now := time.Now()
var uidgen store.UID var uidgen store.UID
log := mlog.New("smtpserver", nil)
message := func(junk bool, ageDays int, ehlo, mailfrom, msgfrom, rcptto string, msgfromvalidation store.Validation, dkimDomains []string, mailfromValid, ehloValid bool, ip string) store.Message { message := func(junk bool, ageDays int, ehlo, mailfrom, msgfrom, rcptto string, msgfromvalidation store.Validation, dkimDomains []string, mailfromValid, ehloValid bool, ip string) store.Message {
mailFromValidation := store.ValidationNone mailFromValidation := store.ValidationNone
@ -77,7 +82,7 @@ func TestReputation(t *testing.T) {
MsgFromLocalpart: msgFrom.Localpart, MsgFromLocalpart: msgFrom.Localpart,
MsgFromDomain: msgFrom.Domain.Name(), MsgFromDomain: msgFrom.Domain.Name(),
MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, msgFrom.Domain).Name(), MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, log.Logger, msgFrom.Domain).Name(),
MailFromValidated: mailfromValid, MailFromValidated: mailfromValid,
EHLOValidated: ehloValid, EHLOValidated: ehloValid,
@ -119,7 +124,7 @@ func TestReputation(t *testing.T) {
rcptToDomain, err := dns.ParseDomain(hm.RcptToDomain) rcptToDomain, err := dns.ParseDomain(hm.RcptToDomain)
tcheck(t, err, "parse rcptToDomain") tcheck(t, err, "parse rcptToDomain")
rcptToOrgDomain := publicsuffix.Lookup(ctxbg, rcptToDomain) rcptToOrgDomain := publicsuffix.Lookup(ctxbg, log.Logger, rcptToDomain)
r := store.Recipient{MessageID: hm.ID, Localpart: hm.RcptToLocalpart, Domain: hm.RcptToDomain, OrgDomain: rcptToOrgDomain.Name(), Sent: hm.Received} r := store.Recipient{MessageID: hm.ID, Localpart: hm.RcptToLocalpart, Domain: hm.RcptToDomain, OrgDomain: rcptToOrgDomain.Name(), Sent: hm.Received}
err = tx.Insert(&r) err = tx.Insert(&r)
tcheck(t, err, "insert recipient") tcheck(t, err, "insert recipient")
@ -136,7 +141,7 @@ func TestReputation(t *testing.T) {
var method reputationMethod var method reputationMethod
err = db.Read(ctxbg, func(tx *bstore.Tx) error { err = db.Read(ctxbg, func(tx *bstore.Tx) error {
var err error var err error
isjunk, conclusive, method, err = reputation(tx, xlog, &m) isjunk, conclusive, method, err = reputation(tx, pkglog, &m)
return err return err
}) })
tcheck(t, err, "read tx") tcheck(t, err, "read tx")

179
smtpserver/server.go
View file

@ -27,6 +27,7 @@ import (
"time" "time"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -57,10 +58,6 @@ import (
"github.com/mjl-/mox/tlsrptdb" "github.com/mjl-/mox/tlsrptdb"
) )
// Most logging should be done through conn.log* functions.
// Only use log in contexts without connection.
var xlog = mlog.New("smtpserver")
// We use panic and recover for error handling while executing commands. // We use panic and recover for error handling while executing commands.
// These errors signal the connection must be closed. // These errors signal the connection must be closed.
var errIO = errors.New("io error") var errIO = errors.New("io error")
@ -233,14 +230,15 @@ func Listen() {
var servers []func() var servers []func()
func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig *tls.Config, submission, xtls bool, maxMessageSize int64, requireTLSForAuth, requireTLSForDelivery, requireTLS bool, dnsBLs []dns.Domain, firstTimeSenderDelay time.Duration) { func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig *tls.Config, submission, xtls bool, maxMessageSize int64, requireTLSForAuth, requireTLSForDelivery, requireTLS bool, dnsBLs []dns.Domain, firstTimeSenderDelay time.Duration) {
log := mlog.New("smtpserver", nil)
addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
if os.Getuid() == 0 { if os.Getuid() == 0 {
xlog.Print("listening for smtp", mlog.Field("listener", name), mlog.Field("address", addr), mlog.Field("protocol", protocol)) log.Print("listening for smtp", slog.String("listener", name), slog.String("address", addr), slog.String("protocol", protocol))
} }
network := mox.Network(ip) network := mox.Network(ip)
ln, err := mox.Listen(network, addr) ln, err := mox.Listen(network, addr)
if err != nil { if err != nil {
xlog.Fatalx("smtp: listen for smtp", err, mlog.Field("protocol", protocol), mlog.Field("listener", name)) log.Fatalx("smtp: listen for smtp", err, slog.String("protocol", protocol), slog.String("listener", name))
} }
if xtls { if xtls {
ln = tls.NewListener(ln, tlsConfig) ln = tls.NewListener(ln, tlsConfig)
@ -250,10 +248,12 @@ func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig
for { for {
conn, err := ln.Accept() conn, err := ln.Accept()
if err != nil { if err != nil {
xlog.Infox("smtp: accept", err, mlog.Field("protocol", protocol), mlog.Field("listener", name)) log.Infox("smtp: accept", err, slog.String("protocol", protocol), slog.String("listener", name))
continue continue
} }
resolver := dns.StrictResolver{} // By leaving Pkg empty, it'll be set by each package that uses the resolver, e.g. spf/dkim/dmarc.
// Package is set on the resolver by the dkim/spf/dmarc/etc packages.
resolver := dns.StrictResolver{Log: log.Logger}
go serve(name, mox.Cid(), hostname, tlsConfig, conn, resolver, submission, xtls, maxMessageSize, requireTLSForAuth, requireTLSForDelivery, requireTLS, dnsBLs, firstTimeSenderDelay) go serve(name, mox.Cid(), hostname, tlsConfig, conn, resolver, submission, xtls, maxMessageSize, requireTLSForAuth, requireTLSForDelivery, requireTLS, dnsBLs, firstTimeSenderDelay)
} }
} }
@ -292,7 +292,7 @@ type conn struct {
localIP net.IP localIP net.IP
remoteIP net.IP remoteIP net.IP
hostname dns.Domain hostname dns.Domain
log *mlog.Log log mlog.Log
maxMessageSize int64 maxMessageSize int64
requireTLSForAuth bool requireTLSForAuth bool
requireTLSForDelivery bool // If set, delivery is only allowed with TLS (STARTTLS), except if delivery is to a TLS reporting address. requireTLSForDelivery bool // If set, delivery is only allowed with TLS (STARTTLS), except if delivery is to a TLS reporting address.
@ -378,7 +378,7 @@ func (c *conn) xcheckAuth() {
} }
} }
func (c *conn) xtrace(level mlog.Level) func() { func (c *conn) xtrace(level slog.Level) func() {
c.xflush() c.xflush()
c.tr.SetTrace(level) c.tr.SetTrace(level)
c.tw.SetTrace(level) c.tw.SetTrace(level)
@ -458,7 +458,7 @@ func (c *conn) Read(buf []byte) (int, error) {
var bufpool = moxio.NewBufpool(8, 2*1024) var bufpool = moxio.NewBufpool(8, 2*1024)
func (c *conn) readline() string { func (c *conn) readline() string {
line, err := bufpool.Readline(c.r) line, err := bufpool.Readline(c.log, c.r)
if err != nil && errors.Is(err, moxio.ErrLineTooLong) { if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
c.writecodeline(smtp.C500BadSyntax, smtp.SeProto5Other0, "line too long, smtp max is 512, we reached 2048", nil) c.writecodeline(smtp.C500BadSyntax, smtp.SeProto5Other0, "line too long, smtp max is 512, we reached 2048", nil)
panic(fmt.Errorf("%s (%w)", err, errIO)) panic(fmt.Errorf("%s (%w)", err, errIO))
@ -476,7 +476,7 @@ func (c *conn) bwritecodeline(code int, secode string, msg string, err error) {
ecode = fmt.Sprintf("%d.%s", code/100, secode) ecode = fmt.Sprintf("%d.%s", code/100, secode)
} }
metricCommands.WithLabelValues(c.kind(), c.cmd, fmt.Sprintf("%d", code), ecode).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) metricCommands.WithLabelValues(c.kind(), c.cmd, fmt.Sprintf("%d", code), ecode).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
c.log.Debugx("smtp command result", err, mlog.Field("kind", c.kind()), mlog.Field("cmd", c.cmd), mlog.Field("code", fmt.Sprintf("%d", code)), mlog.Field("ecode", ecode), mlog.Field("duration", time.Since(c.cmdStart))) c.log.Debugx("smtp command result", err, slog.String("kind", c.kind()), slog.String("cmd", c.cmd), slog.Int("code", code), slog.String("ecode", ecode), slog.Duration("duration", time.Since(c.cmdStart)))
var sep string var sep string
if ecode != "" { if ecode != "" {
@ -563,15 +563,18 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
dnsBLs: dnsBLs, dnsBLs: dnsBLs,
firstTimeSenderDelay: firstTimeSenderDelay, firstTimeSenderDelay: firstTimeSenderDelay,
} }
c.log = xlog.MoreFields(func() []mlog.Pair { var logmutex sync.Mutex
c.log = mlog.New("smtpserver", nil).WithFunc(func() []slog.Attr {
logmutex.Lock()
defer logmutex.Unlock()
now := time.Now() now := time.Now()
l := []mlog.Pair{ l := []slog.Attr{
mlog.Field("cid", c.cid), slog.Int64("cid", c.cid),
mlog.Field("delta", now.Sub(c.lastlog)), slog.Duration("delta", now.Sub(c.lastlog)),
} }
c.lastlog = now c.lastlog = now
if c.username != "" { if c.username != "" {
l = append(l, mlog.Field("username", c.username)) l = append(l, slog.String("username", c.username))
} }
return l return l
}) })
@ -581,7 +584,7 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
c.w = bufio.NewWriter(c.tw) c.w = bufio.NewWriter(c.tw)
metricConnection.WithLabelValues(c.kind()).Inc() metricConnection.WithLabelValues(c.kind()).Inc()
c.log.Info("new connection", mlog.Field("remote", c.conn.RemoteAddr()), mlog.Field("local", c.conn.LocalAddr()), mlog.Field("submission", submission), mlog.Field("tls", tls), mlog.Field("listener", listenerName)) c.log.Info("new connection", slog.Any("remote", c.conn.RemoteAddr()), slog.Any("local", c.conn.LocalAddr()), slog.Bool("submission", submission), slog.Bool("tls", tls), slog.String("listener", listenerName))
defer func() { defer func() {
c.origConn.Close() // Close actual TCP socket, regardless of TLS on top. c.origConn.Close() // Close actual TCP socket, regardless of TLS on top.
@ -599,7 +602,7 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
} else if err, ok := x.(error); ok && isClosed(err) { } else if err, ok := x.(error); ok && isClosed(err) {
c.log.Infox("connection closed", err) c.log.Infox("connection closed", err)
} else { } else {
c.log.Error("unhandled panic", mlog.Field("err", x)) c.log.Error("unhandled panic", slog.Any("err", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Smtpserver) metrics.PanicInc(metrics.Smtpserver)
} }
@ -621,13 +624,13 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
// If remote IP/network resulted in too many authentication failures, refuse to serve. // If remote IP/network resulted in too many authentication failures, refuse to serve.
if submission && !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) { if submission && !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
metrics.AuthenticationRatelimitedInc("submission") metrics.AuthenticationRatelimitedInc("submission")
c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP)) c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many auth failures", nil) c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many auth failures", nil)
return return
} }
if !limiterConnections.Add(c.remoteIP, time.Now(), 1) { if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP)) c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many open connections from your ip or network", nil) c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many open connections from your ip or network", nil)
return return
} }
@ -892,7 +895,7 @@ func (c *conn) cmdStarttls(p *parser) {
} }
cancel() cancel()
tlsversion, ciphersuite := mox.TLSInfo(tlsConn) tlsversion, ciphersuite := mox.TLSInfo(tlsConn)
c.log.Debug("tls server handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite)) c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite))
c.conn = tlsConn c.conn = tlsConn
c.tr = moxio.NewTraceReader(c.log, "RC: ", c) c.tr = moxio.NewTraceReader(c.log, "RC: ", c)
c.tw = moxio.NewTraceWriter(c.log, "LS: ", c) c.tw = moxio.NewTraceWriter(c.log, "LS: ", c)
@ -1030,11 +1033,11 @@ func (c *conn) cmdAuth(p *parser) {
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "cannot assume other role") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "cannot assume other role")
} }
acc, err := store.OpenEmailAuth(authc, password) acc, err := store.OpenEmailAuth(c.log, authc, password)
if err != nil && errors.Is(err, store.ErrUnknownCredentials) { if err != nil && errors.Is(err, store.ErrUnknownCredentials) {
// ../rfc/4954:274 // ../rfc/4954:274
authResult = "badcreds" authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", authc), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", authc), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
} }
xcheckf(err, "verifying credentials") xcheckf(err, "verifying credentials")
@ -1075,11 +1078,11 @@ func (c *conn) cmdAuth(p *parser) {
password := string(xreadContinuation()) password := string(xreadContinuation())
c.xtrace(mlog.LevelTrace) // Restore. c.xtrace(mlog.LevelTrace) // Restore.
acc, err := store.OpenEmailAuth(username, password) acc, err := store.OpenEmailAuth(c.log, username, password)
if err != nil && errors.Is(err, store.ErrUnknownCredentials) { if err != nil && errors.Is(err, store.ErrUnknownCredentials) {
// ../rfc/4954:274 // ../rfc/4954:274
authResult = "badcreds" authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", username), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
} }
xcheckf(err, "verifying credentials") xcheckf(err, "verifying credentials")
@ -1107,11 +1110,11 @@ func (c *conn) cmdAuth(p *parser) {
xsmtpUserErrorf(smtp.C501BadParamSyntax, smtp.SeProto5BadParams4, "malformed cram-md5 response") xsmtpUserErrorf(smtp.C501BadParamSyntax, smtp.SeProto5BadParams4, "malformed cram-md5 response")
} }
addr := t[0] addr := t[0]
c.log.Debug("cram-md5 auth", mlog.Field("address", addr)) c.log.Debug("cram-md5 auth", slog.String("address", addr))
acc, _, err := store.OpenEmail(addr) acc, _, err := store.OpenEmail(c.log, addr)
if err != nil { if err != nil {
if errors.Is(err, store.ErrUnknownCredentials) { if errors.Is(err, store.ErrUnknownCredentials) {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
} }
} }
@ -1127,7 +1130,7 @@ func (c *conn) cmdAuth(p *parser) {
err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error { err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
password, err := bstore.QueryTx[store.Password](tx).Get() password, err := bstore.QueryTx[store.Password](tx).Get()
if err == bstore.ErrAbsent { if err == bstore.ErrAbsent {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
} }
if err != nil { if err != nil {
@ -1141,8 +1144,8 @@ func (c *conn) cmdAuth(p *parser) {
xcheckf(err, "tx read") xcheckf(err, "tx read")
}) })
if ipadhash == nil || opadhash == nil { if ipadhash == nil || opadhash == nil {
c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", mlog.Field("username", addr)) c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", addr))
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
} }
@ -1151,7 +1154,7 @@ func (c *conn) cmdAuth(p *parser) {
opadhash.Write(ipadhash.Sum(nil)) opadhash.Write(ipadhash.Sum(nil))
digest := fmt.Sprintf("%x", opadhash.Sum(nil)) digest := fmt.Sprintf("%x", opadhash.Sum(nil))
if digest != t[1] { if digest != t[1] {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
} }
@ -1181,13 +1184,13 @@ func (c *conn) cmdAuth(p *parser) {
c0 := xreadInitial() c0 := xreadInitial()
ss, err := scram.NewServer(h, c0) ss, err := scram.NewServer(h, c0)
xcheckf(err, "starting scram") xcheckf(err, "starting scram")
c.log.Debug("scram auth", mlog.Field("authentication", ss.Authentication)) c.log.Debug("scram auth", slog.String("authentication", ss.Authentication))
acc, _, err := store.OpenEmail(ss.Authentication) acc, _, err := store.OpenEmail(c.log, ss.Authentication)
if err != nil { if err != nil {
// todo: we could continue scram with a generated salt, deterministically generated // todo: we could continue scram with a generated salt, deterministically generated
// from the username. that way we don't have to store anything but attackers cannot // from the username. that way we don't have to store anything but attackers cannot
// learn if an account exists. same for absent scram saltedpassword below. // learn if an account exists. same for absent scram saltedpassword below.
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible") xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible")
} }
defer func() { defer func() {
@ -1209,8 +1212,8 @@ func (c *conn) cmdAuth(p *parser) {
xscram = password.SCRAMSHA256 xscram = password.SCRAMSHA256
} }
if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) { if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) {
c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", mlog.Field("address", ss.Authentication)) c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("address", ss.Authentication))
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible") xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible")
} }
xcheckf(err, "fetching credentials") xcheckf(err, "fetching credentials")
@ -1230,7 +1233,7 @@ func (c *conn) cmdAuth(p *parser) {
c.readline() // Should be "*" for cancellation. c.readline() // Should be "*" for cancellation.
if errors.Is(err, scram.ErrInvalidProof) { if errors.Is(err, scram.ErrInvalidProof) {
authResult = "badcreds" authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad credentials") xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad credentials")
} }
xcheckf(err, "server final") xcheckf(err, "server final")
@ -1398,11 +1401,11 @@ func (c *conn) cmdMail(p *parser) {
if c.submission && (len(rpath.IPDomain.IP) > 0 || !rpathAllowed()) { if c.submission && (len(rpath.IPDomain.IP) > 0 || !rpathAllowed()) {
// ../rfc/6409:522 // ../rfc/6409:522
c.log.Info("submission with unconfigured mailfrom", mlog.Field("user", c.username), mlog.Field("mailfrom", rpath.String())) c.log.Info("submission with unconfigured mailfrom", slog.String("user", c.username), slog.String("mailfrom", rpath.String()))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user") xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user")
} else if !c.submission && len(rpath.IPDomain.IP) > 0 { } else if !c.submission && len(rpath.IPDomain.IP) > 0 {
// todo future: allow if the IP is the same as this connection is coming from? does later code allow this? // todo future: allow if the IP is the same as this connection is coming from? does later code allow this?
c.log.Info("delivery from address without domain", mlog.Field("mailfrom", rpath.String())) c.log.Info("delivery from address without domain", slog.String("mailfrom", rpath.String()))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7Other0, "domain name required") xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7Other0, "domain name required")
} }
@ -1494,7 +1497,7 @@ func (c *conn) cmdRcpt(p *parser) {
cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid) cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid)
spfctx, spfcancel := context.WithTimeout(cidctx, time.Minute) spfctx, spfcancel := context.WithTimeout(cidctx, time.Minute)
defer spfcancel() defer spfcancel()
receivedSPF, _, _, _, err := spf.Verify(spfctx, c.resolver, spfArgs) receivedSPF, _, _, _, err := spf.Verify(spfctx, c.log.Logger, c.resolver, spfArgs)
spfcancel() spfcancel()
if err != nil { if err != nil {
c.log.Errorx("spf verify for multiple recipients", err) c.log.Errorx("spf verify for multiple recipients", err)
@ -1542,7 +1545,7 @@ func (c *conn) cmdRcpt(p *parser) {
// note: not local for !c.submission is the signal this address is in error. // note: not local for !c.submission is the signal this address is in error.
c.recipients = append(c.recipients, rcptAccount{fpath, false, "", config.Destination{}, ""}) c.recipients = append(c.recipients, rcptAccount{fpath, false, "", config.Destination{}, ""})
} else { } else {
c.log.Errorx("looking up account for delivery", err, mlog.Field("rcptto", fpath)) c.log.Errorx("looking up account for delivery", err, slog.Any("rcptto", fpath))
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "error processing") xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "error processing")
} }
c.bwritecodeline(smtp.C250Completed, smtp.SeAddr1Other0, "now on the list", nil) c.bwritecodeline(smtp.C250Completed, smtp.SeAddr1Other0, "now on the list", nil)
@ -1583,7 +1586,7 @@ func (c *conn) cmdData(p *parser) {
defer c.xtrace(mlog.LevelTracedata)() defer c.xtrace(mlog.LevelTracedata)()
// We read the data into a temporary file. We limit the size and do basic analysis while reading. // We read the data into a temporary file. We limit the size and do basic analysis while reading.
dataFile, err := store.CreateMessageTemp("smtp-deliver") dataFile, err := store.CreateMessageTemp(c.log, "smtp-deliver")
if err != nil { if err != nil {
xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "creating temporary file for message: %s", err) xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "creating temporary file for message: %s", err)
} }
@ -1631,9 +1634,9 @@ func (c *conn) cmdData(p *parser) {
if Localserve && moxvar.Pedantic { if Localserve && moxvar.Pedantic {
// Require that message can be parsed fully. // Require that message can be parsed fully.
p, err := message.Parse(c.log, false, dataFile) p, err := message.Parse(c.log.Logger, false, dataFile)
if err == nil { if err == nil {
err = p.Walk(c.log, nil) err = p.Walk(c.log.Logger, nil)
} }
if err != nil { if err != nil {
// ../rfc/6409:541 // ../rfc/6409:541
@ -1665,9 +1668,9 @@ func (c *conn) cmdData(p *parser) {
iprevStatus, revName, revNames, iprevAuthentic, err = iprev.Lookup(iprevctx, c.resolver, c.remoteIP) iprevStatus, revName, revNames, iprevAuthentic, err = iprev.Lookup(iprevctx, c.resolver, c.remoteIP)
iprevcancel() iprevcancel()
if err != nil { if err != nil {
c.log.Infox("reverse-forward lookup", err, mlog.Field("remoteip", c.remoteIP)) c.log.Infox("reverse-forward lookup", err, slog.Any("remoteip", c.remoteIP))
} }
c.log.Debug("dns iprev check", mlog.Field("addr", c.remoteIP), mlog.Field("status", iprevStatus)) c.log.Debug("dns iprev check", slog.Any("addr", c.remoteIP), slog.Any("status", iprevStatus))
var name string var name string
if revName != "" { if revName != "" {
name = revName name = revName
@ -1721,7 +1724,7 @@ func (c *conn) cmdData(p *parser) {
recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "via", "tcp", "with", with+withComment, "id", mox.ReceivedID(c.cid)) // ../rfc/5321:3158 recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "via", "tcp", "with", with+withComment, "id", mox.ReceivedID(c.cid)) // ../rfc/5321:3158
if c.tls { if c.tls {
tlsConn := c.conn.(*tls.Conn) tlsConn := c.conn.(*tls.Conn)
tlsComment := message.TLSReceivedComment(c.log, tlsConn.ConnectionState()) tlsComment := mox.TLSReceivedComment(c.log, tlsConn.ConnectionState())
recvHdr.Add(" ", tlsComment...) recvHdr.Add(" ", tlsComment...)
} }
recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z)) recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))
@ -1765,10 +1768,10 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
// for other users. // for other users.
// We don't check the Sender field, there is no expectation of verification, ../rfc/7489:2948 // We don't check the Sender field, there is no expectation of verification, ../rfc/7489:2948
// and with Resent headers it seems valid to have someone else as Sender. ../rfc/5322:1578 // and with Resent headers it seems valid to have someone else as Sender. ../rfc/5322:1578
msgFrom, _, header, err := message.From(c.log, true, dataFile) msgFrom, _, header, err := message.From(c.log.Logger, true, dataFile)
if err != nil { if err != nil {
metricSubmission.WithLabelValues("badmessage").Inc() metricSubmission.WithLabelValues("badmessage").Inc()
c.log.Infox("parsing message From address", err, mlog.Field("user", c.username)) c.log.Infox("parsing message From address", err, slog.String("user", c.username))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SeMsg6Other0, "cannot parse header or From address: %v", err) xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SeMsg6Other0, "cannot parse header or From address: %v", err)
} }
accName, _, _, err := mox.FindAccount(msgFrom.Localpart, msgFrom.Domain, true) accName, _, _, err := mox.FindAccount(msgFrom.Localpart, msgFrom.Domain, true)
@ -1778,7 +1781,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
err = mox.ErrAccountNotFound err = mox.ErrAccountNotFound
} }
metricSubmission.WithLabelValues("badfrom").Inc() metricSubmission.WithLabelValues("badfrom").Inc()
c.log.Infox("verifying message From address", err, mlog.Field("user", c.username), mlog.Field("msgfrom", msgFrom)) c.log.Infox("verifying message From address", err, slog.String("user", c.username), slog.Any("msgfrom", msgFrom))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user") xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user")
} }
@ -1835,16 +1838,16 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
// Add DKIM signatures. // Add DKIM signatures.
confDom, ok := mox.Conf.Domain(msgFrom.Domain) confDom, ok := mox.Conf.Domain(msgFrom.Domain)
if !ok { if !ok {
c.log.Error("domain disappeared", mlog.Field("domain", msgFrom.Domain)) c.log.Error("domain disappeared", slog.Any("domain", msgFrom.Domain))
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "internal error") xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "internal error")
} }
dkimConfig := confDom.DKIM dkimConfig := confDom.DKIM
if len(dkimConfig.Sign) > 0 { if len(dkimConfig.Sign) > 0 {
if canonical, err := mox.CanonicalLocalpart(msgFrom.Localpart, confDom); err != nil { if canonical, err := mox.CanonicalLocalpart(msgFrom.Localpart, confDom); err != nil {
c.log.Errorx("determining canonical localpart for dkim signing", err, mlog.Field("localpart", msgFrom.Localpart)) c.log.Errorx("determining canonical localpart for dkim signing", err, slog.Any("localpart", msgFrom.Localpart))
} else if dkimHeaders, err := dkim.Sign(ctx, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil { } else if dkimHeaders, err := dkim.Sign(ctx, c.log.Logger, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil {
c.log.Errorx("dkim sign for domain", err, mlog.Field("domain", msgFrom.Domain)) c.log.Errorx("dkim sign for domain", err, slog.Any("domain", msgFrom.Domain))
metricServerErrors.WithLabelValues("dkimsign").Inc() metricServerErrors.WithLabelValues("dkimsign").Inc()
} else { } else {
msgPrefix = append(msgPrefix, []byte(dkimHeaders)...) msgPrefix = append(msgPrefix, []byte(dkimHeaders)...)
@ -1877,7 +1880,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
mox.Sleep(mox.Context, time.Hour) mox.Sleep(mox.Context, time.Hour)
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out submission due to special localpart") xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out submission due to special localpart")
} else if code != 0 { } else if code != 0 {
c.log.Info("failure due to special localpart", mlog.Field("code", code)) c.log.Info("failure due to special localpart", slog.Int("code", code))
xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code) xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code)
} }
} }
@ -1894,7 +1897,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err) xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err)
} }
metricSubmission.WithLabelValues("ok").Inc() metricSubmission.WithLabelValues("ok").Inc()
c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize)) c.log.Info("message queued for delivery", slog.Any("mailfrom", *c.mailFrom), slog.Any("rcptto", rcptAcc.rcptTo), slog.Bool("smtputf8", c.smtputf8), slog.Int64("msgsize", msgSize))
err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)}) err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
xcheckf(err, "adding outgoing message") xcheckf(err, "adding outgoing message")
@ -1950,7 +1953,7 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) {
mox.Sleep(mox.Context, time.Hour) mox.Sleep(mox.Context, time.Hour)
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out command due to special localpart") xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out command due to special localpart")
} else if code != 0 { } else if code != 0 {
c.log.Info("failure due to special localpart", mlog.Field("code", code)) c.log.Info("failure due to special localpart", slog.Int("code", code))
metricDelivery.WithLabelValues("delivererror", "localserve").Inc() metricDelivery.WithLabelValues("delivererror", "localserve").Inc()
xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code) xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code)
} }
@ -1961,7 +1964,7 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) {
func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, iprevAuthentic bool, dataFile *os.File) { func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, iprevAuthentic bool, dataFile *os.File) {
// todo: in decision making process, if we run into (some) temporary errors, attempt to continue. if we decide to accept, all good. if we decide to reject, we'll make it a temporary reject. // todo: in decision making process, if we run into (some) temporary errors, attempt to continue. if we decide to accept, all good. if we decide to reject, we'll make it a temporary reject.
msgFrom, envelope, headers, err := message.From(c.log, false, dataFile) msgFrom, envelope, headers, err := message.From(c.log.Logger, false, dataFile)
if err != nil { if err != nil {
c.log.Infox("parsing message for From address", err) c.log.Infox("parsing message for From address", err)
} }
@ -2016,7 +2019,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
defer func() { defer func() {
x := recover() // Should not happen, but don't take program down if it does. x := recover() // Should not happen, but don't take program down if it does.
if x != nil { if x != nil {
c.log.Error("dkim verify panic", mlog.Field("err", x)) c.log.Error("dkim verify panic", slog.Any("err", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Dkimverify) metrics.PanicInc(metrics.Dkimverify)
} }
@ -2029,7 +2032,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
dkimctx, dkimcancel := context.WithTimeout(ctx, time.Minute) dkimctx, dkimcancel := context.WithTimeout(ctx, time.Minute)
defer dkimcancel() defer dkimcancel()
// todo future: we could let user configure which dkim headers they require // todo future: we could let user configure which dkim headers they require
dkimResults, dkimErr = dkim.Verify(dkimctx, c.resolver, c.smtputf8, dkim.DefaultPolicy, dataFile, ignoreTestMode) dkimResults, dkimErr = dkim.Verify(dkimctx, c.log.Logger, c.resolver, c.smtputf8, dkim.DefaultPolicy, dataFile, ignoreTestMode)
dkimcancel() dkimcancel()
}() }()
@ -2053,7 +2056,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
defer func() { defer func() {
x := recover() // Should not happen, but don't take program down if it does. x := recover() // Should not happen, but don't take program down if it does.
if x != nil { if x != nil {
c.log.Error("spf verify panic", mlog.Field("err", x)) c.log.Error("spf verify panic", slog.Any("err", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Spfverify) metrics.PanicInc(metrics.Spfverify)
} }
@ -2061,7 +2064,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
defer wg.Done() defer wg.Done()
spfctx, spfcancel := context.WithTimeout(ctx, time.Minute) spfctx, spfcancel := context.WithTimeout(ctx, time.Minute)
defer spfcancel() defer spfcancel()
receivedSPF, spfDomain, spfExpl, spfAuthentic, spfErr = spf.Verify(spfctx, c.resolver, spfArgs) receivedSPF, spfDomain, spfExpl, spfAuthentic, spfErr = spf.Verify(spfctx, c.log.Logger, c.resolver, spfArgs)
spfcancel() spfcancel()
if spfErr != nil { if spfErr != nil {
c.log.Infox("spf verify", spfErr) c.log.Infox("spf verify", spfErr)
@ -2080,7 +2083,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
} }
if nunknown == len(c.recipients) { if nunknown == len(c.recipients) {
// During RCPT TO we found that the address does not exist. // During RCPT TO we found that the address does not exist.
c.log.Info("deliver attempt to unknown user(s)", mlog.Field("recipients", c.recipients)) c.log.Info("deliver attempt to unknown user(s)", slog.Any("recipients", c.recipients))
// Crude attempt to slow down someone trying to guess names. Would work better // Crude attempt to slow down someone trying to guess names. Would work better
// with connection rate limiter. // with connection rate limiter.
@ -2107,7 +2110,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
c.log.Errorx("dkim verify", dkimErr) c.log.Errorx("dkim verify", dkimErr)
authResAddDKIM("none", "", dkimErr.Error(), nil) authResAddDKIM("none", "", dkimErr.Error(), nil)
} else if len(dkimResults) == 0 { } else if len(dkimResults) == 0 {
c.log.Info("no dkim-signature header", mlog.Field("mailfrom", c.mailFrom)) c.log.Info("no dkim-signature header", slog.Any("mailfrom", c.mailFrom))
authResAddDKIM("none", "", "no dkim signatures", nil) authResAddDKIM("none", "", "no dkim signatures", nil)
} }
for i, r := range dkimResults { for i, r := range dkimResults {
@ -2147,7 +2150,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
errmsg = r.Err.Error() errmsg = r.Err.Error()
} }
authResAddDKIM(string(r.Status), comment, errmsg, props) authResAddDKIM(string(r.Status), comment, errmsg, props)
c.log.Debugx("dkim verification result", r.Err, mlog.Field("index", i), mlog.Field("mailfrom", c.mailFrom), mlog.Field("status", r.Status), mlog.Field("domain", domain), mlog.Field("selector", selector), mlog.Field("identity", identity)) c.log.Debugx("dkim verification result", r.Err, slog.Int("index", i), slog.Any("mailfrom", c.mailFrom), slog.Any("status", r.Status), slog.Any("domain", domain), slog.Any("selector", selector), slog.Any("identity", identity))
} }
// Add SPF results to Authentication-Results header. ../rfc/7208:2141 // Add SPF results to Authentication-Results header. ../rfc/7208:2141
@ -2182,7 +2185,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}) })
switch receivedSPF.Result { switch receivedSPF.Result {
case spf.StatusPass: case spf.StatusPass:
c.log.Debug("spf pass", mlog.Field("ip", spfArgs.RemoteIP), mlog.Field("mailfromdomain", spfArgs.MailFromDomain.ASCII)) // todo: log the domain that was actually verified. c.log.Debug("spf pass", slog.Any("ip", spfArgs.RemoteIP), slog.String("mailfromdomain", spfArgs.MailFromDomain.ASCII)) // todo: log the domain that was actually verified.
case spf.StatusFail: case spf.StatusFail:
if spfExpl != "" { if spfExpl != "" {
// Filter out potentially hostile text. ../rfc/7208:2529 // Filter out potentially hostile text. ../rfc/7208:2529
@ -2202,14 +2205,14 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if spfExpl == "" { if spfExpl == "" {
spfExpl = fmt.Sprintf("your ip %s is not on the SPF allowlist for domain %s", spfArgs.RemoteIP, spfDomain.ASCII) spfExpl = fmt.Sprintf("your ip %s is not on the SPF allowlist for domain %s", spfArgs.RemoteIP, spfDomain.ASCII)
} }
c.log.Info("spf fail", mlog.Field("explanation", spfExpl)) // todo future: get this to the client. how? in smtp session in case of a reject due to dmarc fail? c.log.Info("spf fail", slog.String("explanation", spfExpl)) // todo future: get this to the client. how? in smtp session in case of a reject due to dmarc fail?
case spf.StatusTemperror: case spf.StatusTemperror:
c.log.Infox("spf temperror", spfErr) c.log.Infox("spf temperror", spfErr)
case spf.StatusPermerror: case spf.StatusPermerror:
c.log.Infox("spf permerror", spfErr) c.log.Infox("spf permerror", spfErr)
case spf.StatusNone, spf.StatusNeutral, spf.StatusSoftfail: case spf.StatusNone, spf.StatusNeutral, spf.StatusSoftfail:
default: default:
c.log.Error("unknown spf status, treating as None/Neutral", mlog.Field("status", receivedSPF.Result)) c.log.Error("unknown spf status, treating as None/Neutral", slog.Any("status", receivedSPF.Result))
receivedSPF.Result = spf.StatusNone receivedSPF.Result = spf.StatusNone
} }
@ -2228,7 +2231,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
Result: string(dmarcResult.Status), Result: string(dmarcResult.Status),
} }
} else { } else {
msgFromValidation = alignment(ctx, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity) msgFromValidation = alignment(ctx, c.log, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity)
// We are doing the DMARC evaluation now. But we only store it for inclusion in an // We are doing the DMARC evaluation now. But we only store it for inclusion in an
// aggregate report when we actually use it. We use an evaluation for each // aggregate report when we actually use it. We use an evaluation for each
@ -2241,7 +2244,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute) dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute)
defer dmarccancel() defer dmarccancel()
dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage) dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.log.Logger, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage)
dmarccancel() dmarccancel()
var comment string var comment string
if dmarcResult.RecordAuthentic { if dmarcResult.RecordAuthentic {
@ -2265,7 +2268,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507 // todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507
} }
c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain)) c.log.Debug("dmarc verification", slog.Any("result", dmarcResult.Status), slog.Any("domain", msgFrom.Domain))
// Prepare for analyzing content, calculating reputation. // Prepare for analyzing content, calculating reputation.
ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP) ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP)
@ -2304,13 +2307,13 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
var deliverErrors []deliverError var deliverErrors []deliverError
addError := func(rcptAcc rcptAccount, code int, secode string, userError bool, errmsg string) { addError := func(rcptAcc rcptAccount, code int, secode string, userError bool, errmsg string) {
e := deliverError{rcptAcc.rcptTo, code, secode, userError, errmsg} e := deliverError{rcptAcc.rcptTo, code, secode, userError, errmsg}
c.log.Info("deliver error", mlog.Field("rcptto", e.rcptTo), mlog.Field("code", code), mlog.Field("secode", "secode"), mlog.Field("usererror", userError), mlog.Field("errmsg", errmsg)) c.log.Info("deliver error", slog.Any("rcptto", e.rcptTo), slog.Int("code", code), slog.String("secode", "secode"), slog.Bool("usererror", userError), slog.String("errmsg", errmsg))
deliverErrors = append(deliverErrors, e) deliverErrors = append(deliverErrors, e)
} }
// For each recipient, do final spam analysis and delivery. // For each recipient, do final spam analysis and delivery.
for _, rcptAcc := range c.recipients { for _, rcptAcc := range c.recipients {
log := c.log.Fields(mlog.Field("mailfrom", c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo)) log := c.log.With(slog.Any("mailfrom", c.mailFrom), slog.Any("rcptto", rcptAcc.rcptTo))
// If this is not a valid local user, we send back a DSN. This can only happen when // If this is not a valid local user, we send back a DSN. This can only happen when
// there are also valid recipients, and only when remote is SPF-verified, so the DSN // there are also valid recipients, and only when remote is SPF-verified, so the DSN
@ -2326,9 +2329,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
continue continue
} }
acc, err := store.OpenAccount(rcptAcc.accountName) acc, err := store.OpenAccount(log, rcptAcc.accountName)
if err != nil { if err != nil {
log.Errorx("open account", err, mlog.Field("account", rcptAcc.accountName)) log.Errorx("open account", err, slog.Any("account", rcptAcc.accountName))
metricDelivery.WithLabelValues("accounterror", "").Inc() metricDelivery.WithLabelValues("accounterror", "").Inc()
addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing") addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing")
continue continue
@ -2347,7 +2350,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
err = acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) { err = acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) {
now := time.Now() now := time.Now()
defer func() { defer func() {
log.Debugx("checking message and size delivery rates", retErr, mlog.Field("duration", time.Since(now))) log.Debugx("checking message and size delivery rates", retErr, slog.Duration("duration", time.Since(now)))
}() }()
checkCount := func(msg store.Message, window time.Duration, limit int) { checkCount := func(msg store.Message, window time.Duration, limit int) {
@ -2440,7 +2443,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
RcptToDomain: rcptAcc.rcptTo.IPDomain.Domain.Name(), RcptToDomain: rcptAcc.rcptTo.IPDomain.Domain.Name(),
MsgFromLocalpart: msgFrom.Localpart, MsgFromLocalpart: msgFrom.Localpart,
MsgFromDomain: msgFrom.Domain.Name(), MsgFromDomain: msgFrom.Domain.Name(),
MsgFromOrgDomain: publicsuffix.Lookup(ctx, msgFrom.Domain).Name(), MsgFromOrgDomain: publicsuffix.Lookup(ctx, log.Logger, msgFrom.Domain).Name(),
EHLOValidated: ehloValidation == store.ValidationPass, EHLOValidated: ehloValidation == store.ValidationPass,
MailFromValidated: mailFromValidation == store.ValidationPass, MailFromValidated: mailFromValidation == store.ValidationPass,
MsgFromValidated: msgFromValidation == store.ValidationStrict || msgFromValidation == store.ValidationDMARC || msgFromValidation == store.ValidationRelaxed, MsgFromValidated: msgFromValidation == store.ValidationStrict || msgFromValidation == store.ValidationDMARC || msgFromValidation == store.ValidationRelaxed,
@ -2616,7 +2619,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// We'll include all signatures for the organizational domain, even if they weren't // We'll include all signatures for the organizational domain, even if they weren't
// relevant due to strict alignment requirement. // relevant due to strict alignment requirement.
for _, dkimResult := range dkimResults { for _, dkimResult := range dkimResults {
if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, msgFrom.Domain) != publicsuffix.Lookup(ctx, dkimResult.Sig.Domain) { if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, log.Logger, msgFrom.Domain) != publicsuffix.Lookup(ctx, log.Logger, dkimResult.Sig.Domain) {
continue continue
} }
r := dmarcrpt.DKIMAuthResult{ r := dmarcrpt.DKIMAuthResult{
@ -2684,7 +2687,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
} }
} }
log.Info("incoming message rejected", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom)) log.Info("incoming message rejected", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom))
metricDelivery.WithLabelValues("reject", a.reason).Inc() metricDelivery.WithLabelValues("reject", a.reason).Inc()
c.setSlow(true) c.setSlow(true)
addError(rcptAcc, a.code, a.secode, a.userError, a.errmsg) addError(rcptAcc, a.code, a.secode, a.userError, a.errmsg)
@ -2704,7 +2707,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
} }
if a.tlsReport != nil { if a.tlsReport != nil {
// todo future: add rate limiting to prevent DoS attacks. // todo future: add rate limiting to prevent DoS attacks.
if err := tlsrptdb.AddReport(ctx, msgFrom.Domain, c.mailFrom.String(), rcptAcc.destination.HostTLSReports, a.tlsReport); err != nil { if err := tlsrptdb.AddReport(ctx, c.log, msgFrom.Domain, c.mailFrom.String(), rcptAcc.destination.HostTLSReports, a.tlsReport); err != nil {
log.Errorx("saving TLSRPT report in database", err) log.Errorx("saving TLSRPT report in database", err)
} else { } else {
log.Info("tlsrpt report processed") log.Info("tlsrpt report processed")
@ -2717,13 +2720,13 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// delivering. If this turns out to be a spammer, we've kept one of their // delivering. If this turns out to be a spammer, we've kept one of their
// connections busy. // connections busy.
if delayFirstTime && !m.IsForward && a.reason == reasonNoBadSignals && c.firstTimeSenderDelay > 0 { if delayFirstTime && !m.IsForward && a.reason == reasonNoBadSignals && c.firstTimeSenderDelay > 0 {
log.Debug("delaying before delivering from sender without reputation", mlog.Field("delay", c.firstTimeSenderDelay)) log.Debug("delaying before delivering from sender without reputation", slog.Duration("delay", c.firstTimeSenderDelay))
mox.Sleep(mox.Context, c.firstTimeSenderDelay) mox.Sleep(mox.Context, c.firstTimeSenderDelay)
} }
// Gather the message-id before we deliver and the file may be consumed. // Gather the message-id before we deliver and the file may be consumed.
if !parsedMessageID { if !parsedMessageID {
if p, err := message.Parse(c.log, false, store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil { if p, err := message.Parse(c.log.Logger, false, store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil {
log.Infox("parsing message for message-id", err) log.Infox("parsing message for message-id", err)
} else if header, err := p.Header(); err != nil { } else if header, err := p.Header(); err != nil {
log.Infox("parsing message header for message-id", err) log.Infox("parsing message header for message-id", err)
@ -2735,11 +2738,11 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if Localserve { if Localserve {
code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart) code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart)
if timeout { if timeout {
c.log.Info("timing out due to special localpart") log.Info("timing out due to special localpart")
mox.Sleep(mox.Context, time.Hour) mox.Sleep(mox.Context, time.Hour)
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeOther00}, "timing out delivery due to special localpart") xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeOther00}, "timing out delivery due to special localpart")
} else if code != 0 { } else if code != 0 {
c.log.Info("failure due to special localpart", mlog.Field("code", code)) log.Info("failure due to special localpart", slog.Int("code", code))
metricDelivery.WithLabelValues("delivererror", "localserve").Inc() metricDelivery.WithLabelValues("delivererror", "localserve").Inc()
addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code)) addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code))
} }
@ -2752,12 +2755,12 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
return return
} }
metricDelivery.WithLabelValues("delivered", a.reason).Inc() metricDelivery.WithLabelValues("delivered", a.reason).Inc()
log.Info("incoming message delivered", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom)) log.Info("incoming message delivered", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom))
conf, _ := acc.Conf() conf, _ := acc.Conf()
if conf.RejectsMailbox != "" && m.MessageID != "" { if conf.RejectsMailbox != "" && m.MessageID != "" {
if err := acc.RejectsRemove(log, conf.RejectsMailbox, m.MessageID); err != nil { if err := acc.RejectsRemove(log, conf.RejectsMailbox, m.MessageID); err != nil {
log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID)) log.Errorx("removing message from rejects mailbox", err, slog.String("messageid", messageID))
} }
} }
}) })

26
smtpserver/server_test.go
View file

@ -23,6 +23,8 @@ import (
"testing" "testing"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/config" "github.com/mjl-/mox/config"
@ -106,15 +108,16 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test
dmarcdb.EvalDB = nil dmarcdb.EvalDB = nil
} }
log := mlog.New("smtpserver", nil)
mox.Context = ctxbg mox.Context = ctxbg
mox.ConfigStaticPath = configPath mox.ConfigStaticPath = configPath
mox.MustLoadConfig(true, false) mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
os.RemoveAll(dataDir) os.RemoveAll(dataDir)
var err error var err error
ts.acc, err = store.OpenAccount("mjl") ts.acc, err = store.OpenAccount(log, "mjl")
tcheck(t, err, "open account") tcheck(t, err, "open account")
err = ts.acc.SetPassword("testtest") err = ts.acc.SetPassword(log, "testtest")
tcheck(t, err, "set password") tcheck(t, err, "set password")
ts.switchStop = store.Switchboard() ts.switchStop = store.Switchboard()
err = queue.Init() err = queue.Init()
@ -169,7 +172,8 @@ func (ts *testserver) run(fn func(helloErr error, client *smtpclient.Client)) {
Auth: auth, Auth: auth,
RootCAs: mox.Conf.Static.TLS.CertPool, RootCAs: mox.Conf.Static.TLS.CertPool,
} }
client, err := smtpclient.New(ctxbg, xlog.WithCid(ts.cid-1), clientConn, ts.tlsmode, ts.tlspkix, ourHostname, remoteHostname, opts) log := pkglog.WithCid(ts.cid - 1)
client, err := smtpclient.New(ctxbg, log.Logger, clientConn, ts.tlsmode, ts.tlspkix, ourHostname, remoteHostname, opts)
if err != nil { if err != nil {
clientConn.Close() clientConn.Close()
} else { } else {
@ -355,13 +359,13 @@ func TestDelivery(t *testing.T) {
} }
func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) { func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) {
mf, err := store.CreateMessageTemp("queue-dsn") mf, err := store.CreateMessageTemp(pkglog, "queue-dsn")
tcheck(t, err, "temp message") tcheck(t, err, "temp message")
defer os.Remove(mf.Name()) defer os.Remove(mf.Name())
defer mf.Close() defer mf.Close()
_, err = mf.Write([]byte(msg)) _, err = mf.Write([]byte(msg))
tcheck(t, err, "write message") tcheck(t, err, "write message")
err = acc.DeliverMailbox(xlog, mailbox, m, mf) err = acc.DeliverMailbox(pkglog, mailbox, m, mf)
tcheck(t, err, "deliver message") tcheck(t, err, "deliver message")
err = mf.Close() err = mf.Close()
tcheck(t, err, "close message") tcheck(t, err, "close message")
@ -376,7 +380,7 @@ func tretrain(t *testing.T, acc *store.Account) {
bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom") bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom")
os.Remove(dbPath) os.Remove(dbPath)
os.Remove(bloomPath) os.Remove(bloomPath)
jf, _, err := acc.OpenJunkFilter(ctxbg, xlog) jf, _, err := acc.OpenJunkFilter(ctxbg, pkglog)
tcheck(t, err, "open junk filter") tcheck(t, err, "open junk filter")
defer jf.Close() defer jf.Close()
@ -1004,7 +1008,7 @@ func TestTLSReport(t *testing.T) {
tcheck(t, xerr, "write msg") tcheck(t, xerr, "write msg")
msg := msgb.String() msg := msgb.String()
headers, xerr := dkim.Sign(ctxbg, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg)) headers, xerr := dkim.Sign(ctxbg, pkglog.Logger, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg))
tcheck(t, xerr, "dkim sign") tcheck(t, xerr, "dkim sign")
msg = headers + msg msg = headers + msg
@ -1040,7 +1044,7 @@ func TestRatelimitConnectionrate(t *testing.T) {
// We'll be creating 300 connections, no TLS and reduce noise. // We'll be creating 300 connections, no TLS and reduce noise.
ts.tlsmode = smtpclient.TLSSkip ts.tlsmode = smtpclient.TLSSkip
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelInfo}) mlog.SetConfig(map[string]slog.Level{"": mlog.LevelInfo})
// We may be passing a window boundary during this tests. The limit is 300/minute. // We may be passing a window boundary during this tests. The limit is 300/minute.
// So make twice that many connections and hope the tests don't take too long. // So make twice that many connections and hope the tests don't take too long.
@ -1272,7 +1276,7 @@ func TestCatchall(t *testing.T) {
tcheck(t, err, "checking delivered messages") tcheck(t, err, "checking delivered messages")
tcompare(t, n, 3) tcompare(t, n, 3)
acc, err := store.OpenAccount("catchall") acc, err := store.OpenAccount(pkglog, "catchall")
tcheck(t, err, "open account") tcheck(t, err, "open account")
defer acc.Close() defer acc.Close()
n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count() n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count()
@ -1361,7 +1365,7 @@ test email
f, err := queue.OpenMessage(ctxbg, msgs[0].ID) f, err := queue.OpenMessage(ctxbg, msgs[0].ID)
tcheck(t, err, "open message in queue") tcheck(t, err, "open message in queue")
defer f.Close() defer f.Close()
results, err := dkim.Verify(ctxbg, resolver, false, dkim.DefaultPolicy, f, false) results, err := dkim.Verify(ctxbg, pkglog.Logger, resolver, false, dkim.DefaultPolicy, f, false)
tcheck(t, err, "verifying dkim message") tcheck(t, err, "verifying dkim message")
tcompare(t, len(results), 1) tcompare(t, len(results), 1)
tcompare(t, results[0].Status, dkim.StatusPass) tcompare(t, results[0].Status, dkim.StatusPass)
@ -1504,7 +1508,7 @@ test email
tcheck(t, err, "listing queue") tcheck(t, err, "listing queue")
tcompare(t, len(msgs), 1) tcompare(t, len(msgs), 1)
tcompare(t, msgs[0].RequireTLS, expRequireTLS) tcompare(t, msgs[0].RequireTLS, expRequireTLS)
_, err = queue.Drop(ctxbg, msgs[0].ID, "", "") _, err = queue.Drop(ctxbg, pkglog, msgs[0].ID, "", "")
tcheck(t, err, "deleting message from queue") tcheck(t, err, "deleting message from queue")
}) })
} }

42
spf/spf.go
View file

@ -16,6 +16,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
@ -28,8 +30,6 @@ import (
// sure we make names absolute when looking up. For verifying, we do not want to // sure we make names absolute when looking up. For verifying, we do not want to
// verify names relative to our local search domain. // verify names relative to our local search domain.
var xlog = mlog.New("spf")
var ( var (
metricSPFVerify = promauto.NewHistogramVec( metricSPFVerify = promauto.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
@ -129,11 +129,11 @@ var timeNow = time.Now
// Lookup looks up and parses an SPF TXT record for domain. // Lookup looks up and parses an SPF TXT record for domain.
// //
// authentic indicates if the DNS results were DNSSEC-verified. // authentic indicates if the DNS results were DNSSEC-verified.
func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rstatus Status, rtxt string, rrecord *Record, authentic bool, rerr error) { func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rstatus Status, rtxt string, rrecord *Record, authentic bool, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("spf", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("spf lookup result", rerr, mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) log.Debugx("spf lookup result", rerr, slog.Any("domain", domain), slog.Any("status", rstatus), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start)))
}() }()
// ../rfc/7208:586 // ../rfc/7208:586
@ -194,12 +194,12 @@ func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rsta
// of 2 lookups resulting in no records ("void lookups"). // of 2 lookups resulting in no records ("void lookups").
// //
// authentic indicates if the DNS results were DNSSEC-verified. // authentic indicates if the DNS results were DNSSEC-verified.
func Verify(ctx context.Context, resolver dns.Resolver, args Args) (received Received, domain dns.Domain, explanation string, authentic bool, rerr error) { func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, args Args) (received Received, domain dns.Domain, explanation string, authentic bool, rerr error) {
log := xlog.WithContext(ctx) log := mlog.New("spf", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
metricSPFVerify.WithLabelValues(string(received.Result)).Observe(float64(time.Since(start)) / float64(time.Second)) metricSPFVerify.WithLabelValues(string(received.Result)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("spf verify result", rerr, mlog.Field("domain", args.domain), mlog.Field("ip", args.RemoteIP), mlog.Field("status", received.Result), mlog.Field("explanation", explanation), mlog.Field("duration", time.Since(start))) log.Debugx("spf verify result", rerr, slog.Any("domain", args.domain), slog.Any("ip", args.RemoteIP), slog.Any("status", received.Result), slog.String("explanation", explanation), slog.Duration("duration", time.Since(start)))
}() }()
isHello, ok := prepare(&args) isHello, ok := prepare(&args)
@ -215,7 +215,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, args Args) (received Rec
return received, dns.Domain{}, "", false, nil return received, dns.Domain{}, "", false, nil
} }
status, mechanism, expl, authentic, err := checkHost(ctx, resolver, args) status, mechanism, expl, authentic, err := checkHost(ctx, log, resolver, args)
comment := fmt.Sprintf("domain %s", args.domain.ASCII) comment := fmt.Sprintf("domain %s", args.domain.ASCII)
if isHello { if isHello {
comment += ", from ehlo because mailfrom is empty" comment += ", from ehlo because mailfrom is empty"
@ -272,37 +272,35 @@ func prepare(args *Args) (isHello bool, ok bool) {
} }
// lookup spf record, then evaluate args against it. // lookup spf record, then evaluate args against it.
func checkHost(ctx context.Context, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { func checkHost(ctx context.Context, log mlog.Log, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
status, _, record, rauthentic, err := Lookup(ctx, resolver, args.domain) status, _, record, rauthentic, err := Lookup(ctx, log.Logger, resolver, args.domain)
if err != nil { if err != nil {
return status, "", "", rauthentic, err return status, "", "", rauthentic, err
} }
var evalAuthentic bool var evalAuthentic bool
rstatus, mechanism, rexplanation, evalAuthentic, rerr = evaluate(ctx, record, resolver, args) rstatus, mechanism, rexplanation, evalAuthentic, rerr = evaluate(ctx, log, record, resolver, args)
rauthentic = rauthentic && evalAuthentic rauthentic = rauthentic && evalAuthentic
return return
} }
// Evaluate evaluates the IP and names from args against the SPF DNS record for the domain. // Evaluate evaluates the IP and names from args against the SPF DNS record for the domain.
func Evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { func Evaluate(ctx context.Context, elog *slog.Logger, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
log := mlog.New("spf", elog)
_, ok := prepare(&args) _, ok := prepare(&args)
if !ok { if !ok {
return StatusNone, "default", "", false, fmt.Errorf("no domain name to validate") return StatusNone, "default", "", false, fmt.Errorf("no domain name to validate")
} }
return evaluate(ctx, record, resolver, args) return evaluate(ctx, log, record, resolver, args)
} }
// evaluate RemoteIP against domain from args, given record. // evaluate RemoteIP against domain from args, given record.
func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { func evaluate(ctx context.Context, log mlog.Log, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
log := xlog.WithContext(ctx)
start := time.Now() start := time.Now()
defer func() { defer func() {
log.Debugx("spf evaluate result", rerr, mlog.Field("dnsrequests", *args.dnsRequests), mlog.Field("voidlookups", *args.voidLookups), mlog.Field("domain", args.domain), mlog.Field("status", rstatus), mlog.Field("mechanism", mechanism), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start))) log.Debugx("spf evaluate result", rerr, slog.Int("dnsrequests", *args.dnsRequests), slog.Int("voidlookups", *args.voidLookups), slog.Any("domain", args.domain), slog.Any("status", rstatus), slog.String("mechanism", mechanism), slog.String("explanation", rexplanation), slog.Duration("duration", time.Since(start)))
}() }()
resolver = dns.WithPackage(resolver, "spf")
if args.dnsRequests == nil { if args.dnsRequests == nil {
args.dnsRequests = new(int) args.dnsRequests = new(int)
args.voidLookups = new(int) args.voidLookups = new(int)
@ -388,7 +386,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A
nargs := args nargs := args
nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")} nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")}
nargs.explanation = &record.Explanation // ../rfc/7208:1548 nargs.explanation = &record.Explanation // ../rfc/7208:1548
status, _, _, authentic, err := checkHost(ctx, resolver, nargs) status, _, _, authentic, err := checkHost(ctx, log, resolver, nargs)
rauthentic = rauthentic && authentic rauthentic = rauthentic && authentic
// ../rfc/7208:1202 // ../rfc/7208:1202
switch status { switch status {
@ -477,7 +475,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A
for _, rname := range rnames { for _, rname := range rnames {
rd, err := dns.ParseDomain(strings.TrimSuffix(rname, ".")) rd, err := dns.ParseDomain(strings.TrimSuffix(rname, "."))
if err != nil { if err != nil {
log.Errorx("bad address in ptr record", err, mlog.Field("address", rname)) log.Errorx("bad address in ptr record", err, slog.String("address", rname))
continue continue
} }
// ../rfc/7208-eid4751 ../rfc/7208:1323 // ../rfc/7208-eid4751 ../rfc/7208:1323
@ -565,7 +563,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A
nargs := args nargs := args
nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")} nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")}
nargs.explanation = nil // ../rfc/7208:1548 nargs.explanation = nil // ../rfc/7208:1548
status, mechanism, expl, authentic, err := checkHost(ctx, resolver, nargs) status, mechanism, expl, authentic, err := checkHost(ctx, log, resolver, nargs)
rauthentic = rauthentic && authentic rauthentic = rauthentic && authentic
if status == StatusNone { if status == StatusNone {
return StatusPermerror, mechanism, "", rauthentic, err return StatusPermerror, mechanism, "", rauthentic, err

15
spf/spf_test.go
View file

@ -10,9 +10,12 @@ import (
"time" "time"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
var pkglog = mlog.New("spf", nil)
func TestLookup(t *testing.T) { func TestLookup(t *testing.T) {
resolver := dns.MockResolver{ resolver := dns.MockResolver{
TXT: map[string][]string{ TXT: map[string][]string{
@ -31,7 +34,7 @@ func TestLookup(t *testing.T) {
t.Helper() t.Helper()
d := dns.Domain{ASCII: domain} d := dns.Domain{ASCII: domain}
status, txt, record, _, err := Lookup(context.Background(), resolver, d) status, txt, record, _, err := Lookup(context.Background(), pkglog.Logger, resolver, d)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %v, expected err %v", err, expErr) t.Fatalf("got err %v, expected err %v", err, expErr)
} }
@ -259,7 +262,7 @@ func TestVerify(t *testing.T) {
LocalIP: xip("127.0.0.1"), LocalIP: xip("127.0.0.1"),
LocalHostname: dns.Domain{ASCII: "localhost"}, LocalHostname: dns.Domain{ASCII: "localhost"},
} }
received, _, _, _, err := Verify(ctx, r, args) received, _, _, _, err := Verify(ctx, pkglog.Logger, r, args)
if received.Result != status { if received.Result != status {
t.Fatalf("got status %q, expected %q, for ip %q (err %v)", received.Result, status, ip, err) t.Fatalf("got status %q, expected %q, for ip %q (err %v)", received.Result, status, ip, err)
} }
@ -345,7 +348,7 @@ func TestVerifyMultipleDomain(t *testing.T) {
LocalIP: net.ParseIP("127.0.0.1"), LocalIP: net.ParseIP("127.0.0.1"),
LocalHostname: dns.Domain{ASCII: "localhost"}, LocalHostname: dns.Domain{ASCII: "localhost"},
} }
received, _, _, _, err := Verify(context.Background(), resolver, args) received, _, _, _, err := Verify(context.Background(), pkglog.Logger, resolver, args)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %s", err) t.Fatalf("unexpected error: %s", err)
} }
@ -370,7 +373,7 @@ func TestVerifyScenarios(t *testing.T) {
test := func(resolver dns.Resolver, args Args, expStatus Status, expDomain string, expExpl string, expErr error) { test := func(resolver dns.Resolver, args Args, expStatus Status, expDomain string, expExpl string, expErr error) {
t.Helper() t.Helper()
recv, d, expl, _, err := Verify(context.Background(), resolver, args) recv, d, expl, _, err := Verify(context.Background(), pkglog.Logger, resolver, args)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %v, expected %v", err, expErr) t.Fatalf("got err %v, expected %v", err, expErr)
} }
@ -505,7 +508,7 @@ func TestEvaluate(t *testing.T) {
record := &Record{} record := &Record{}
resolver := dns.MockResolver{} resolver := dns.MockResolver{}
args := Args{} args := Args{}
status, _, _, _, _ := Evaluate(context.Background(), record, resolver, args) status, _, _, _, _ := Evaluate(context.Background(), pkglog.Logger, record, resolver, args)
if status != StatusNone { if status != StatusNone {
t.Fatalf("got status %q, expected none", status) t.Fatalf("got status %q, expected none", status)
} }
@ -513,7 +516,7 @@ func TestEvaluate(t *testing.T) {
args = Args{ args = Args{
HelloDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "test.example"}}, HelloDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "test.example"}},
} }
status, mechanism, _, _, err := Evaluate(context.Background(), record, resolver, args) status, mechanism, _, _, err := Evaluate(context.Background(), pkglog.Logger, record, resolver, args)
if status != StatusNeutral || mechanism != "default" || err != nil { if status != StatusNeutral || mechanism != "default" || err != nil {
t.Fatalf("got status %q, mechanism %q, err %v, expected neutral, default, no error", status, mechanism, err) t.Fatalf("got status %q, mechanism %q, err %v, expected neutral, default, no error", status, mechanism, err)
} }

93
store/account.go
View file

@ -44,6 +44,7 @@ import (
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"golang.org/x/text/unicode/norm" "golang.org/x/text/unicode/norm"
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
@ -67,8 +68,6 @@ import (
// false again. // false again.
var CheckConsistencyOnClose = true var CheckConsistencyOnClose = true
var xlog = mlog.New("store")
var ( var (
ErrUnknownMailbox = errors.New("no such mailbox") ErrUnknownMailbox = errors.New("no such mailbox")
ErrUnknownCredentials = errors.New("credentials not found") ErrUnknownCredentials = errors.New("credentials not found")
@ -577,15 +576,15 @@ func (m *Message) PrepareExpunge() {
// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the // PrepareThreading sets MessageID and SubjectBase (used in threading) based on the
// envelope in part. // envelope in part.
func (m *Message) PrepareThreading(log *mlog.Log, part *message.Part) { func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {
if part.Envelope == nil { if part.Envelope == nil {
return return
} }
messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID) messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)
if err != nil { if err != nil {
log.Debugx("parsing message-id, ignoring", err, mlog.Field("messageid", part.Envelope.MessageID)) log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))
} else if raw { } else if raw {
log.Debug("could not parse message-id as address, continuing with raw value", mlog.Field("messageid", part.Envelope.MessageID)) log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))
} }
m.MessageID = messageID m.MessageID = messageID
m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false) m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)
@ -747,7 +746,7 @@ func closeAccount(acc *Account) (rerr error) {
// //
// No additional data path prefix or ".db" suffix should be added to the name. // No additional data path prefix or ".db" suffix should be added to the name.
// A single shared account exists per name. // A single shared account exists per name.
func OpenAccount(name string) (*Account, error) { func OpenAccount(log mlog.Log, name string) (*Account, error) {
openAccounts.Lock() openAccounts.Lock()
defer openAccounts.Unlock() defer openAccounts.Unlock()
if acc, ok := openAccounts.names[name]; ok { if acc, ok := openAccounts.names[name]; ok {
@ -759,7 +758,7 @@ func OpenAccount(name string) (*Account, error) {
return nil, ErrAccountUnknown return nil, ErrAccountUnknown
} }
acc, err := openAccount(name) acc, err := openAccount(log, name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -768,15 +767,15 @@ func OpenAccount(name string) (*Account, error) {
} }
// openAccount opens an existing account, or creates it if it is missing. // openAccount opens an existing account, or creates it if it is missing.
func openAccount(name string) (a *Account, rerr error) { func openAccount(log mlog.Log, name string) (a *Account, rerr error) {
dir := filepath.Join(mox.DataDirPath("accounts"), name) dir := filepath.Join(mox.DataDirPath("accounts"), name)
return OpenAccountDB(dir, name) return OpenAccountDB(log, dir, name)
} }
// OpenAccountDB opens an account database file and returns an initialized account // OpenAccountDB opens an account database file and returns an initialized account
// or error. Only exported for use by subcommands that verify the database file. // or error. Only exported for use by subcommands that verify the database file.
// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth. // Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.
func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {
dbpath := filepath.Join(accountDir, "index.db") dbpath := filepath.Join(accountDir, "index.db")
// Create account if it doesn't exist yet. // Create account if it doesn't exist yet.
@ -823,7 +822,7 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error { return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {
if !mentioned { if !mentioned {
mentioned = true mentioned = true
xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName)) log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))
} }
mc, err := mb.CalculateCounts(tx) mc, err := mb.CalculateCounts(tx)
if err != nil { if err != nil {
@ -866,17 +865,17 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
// Ensure all messages have a MessageID and SubjectBase, which are needed when // Ensure all messages have a MessageID and SubjectBase, which are needed when
// matching threads. // matching threads.
// Then assign messages to threads, in the same way we do during imports. // Then assign messages to threads, in the same way we do during imports.
xlog.Info("upgrading account for threading, in background", mlog.Field("account", acc.Name)) log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))
go func() { go func() {
defer func() { defer func() {
err := closeAccount(acc) err := closeAccount(acc)
xlog.Check(err, "closing use of account after upgrading account storage for threads", mlog.Field("account", a.Name)) log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))
}() }()
defer func() { defer func() {
x := recover() // Should not happen, but don't take program down if it does. x := recover() // Should not happen, but don't take program down if it does.
if x != nil { if x != nil {
xlog.Error("upgradeThreads panic", mlog.Field("err", x)) log.Error("upgradeThreads panic", slog.Any("err", x))
debug.PrintStack() debug.PrintStack()
metrics.PanicInc(metrics.Upgradethreads) metrics.PanicInc(metrics.Upgradethreads)
acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x) acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)
@ -886,12 +885,12 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
close(acc.threadsCompleted) close(acc.threadsCompleted)
}() }()
err := upgradeThreads(mox.Shutdown, acc, &up) err := upgradeThreads(mox.Shutdown, log, acc, &up)
if err != nil { if err != nil {
a.threadsErr = err a.threadsErr = err
xlog.Errorx("upgrading account for threading, aborted", err, mlog.Field("account", a.Name)) log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))
} else { } else {
xlog.Info("upgrading account for threading, completed", mlog.Field("account", a.Name)) log.Info("upgrading account for threading, completed", slog.String("account", a.Name))
} }
}() }()
return acc, nil return acc, nil
@ -901,7 +900,7 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
// account has completed, and returns an error if not successful. // account has completed, and returns an error if not successful.
// //
// To be used before starting an import of messages. // To be used before starting an import of messages.
func (a *Account) ThreadingWait(log *mlog.Log) error { func (a *Account) ThreadingWait(log mlog.Log) error {
select { select {
case <-a.threadsCompleted: case <-a.threadsCompleted:
return a.threadsErr return a.threadsErr
@ -1224,7 +1223,7 @@ func (a *Account) WithRLock(fn func()) {
// Caller must broadcast new message. // Caller must broadcast new message.
// //
// Caller must update mailbox counts. // Caller must update mailbox counts.
func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error { func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error {
if m.Expunged { if m.Expunged {
return fmt.Errorf("cannot deliver expunged message") return fmt.Errorf("cannot deliver expunged message")
} }
@ -1245,9 +1244,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile. mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.
var part *message.Part var part *message.Part
if m.ParsedBuf == nil { if m.ParsedBuf == nil {
p, err := message.EnsurePart(log, false, mr, m.Size) p, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil { if err != nil {
log.Infox("parsing delivered message", err, mlog.Field("parse", ""), mlog.Field("message", m.ID)) log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))
// We continue, p is still valid. // We continue, p is still valid.
} }
part = &p part = &p
@ -1259,7 +1258,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
} else { } else {
var p message.Part var p message.Part
if err := json.Unmarshal(m.ParsedBuf, &p); err != nil { if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
log.Errorx("unmarshal parsed message, continuing", err, mlog.Field("parse", "")) log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))
} else { } else {
part = &p part = &p
} }
@ -1328,19 +1327,19 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
for _, addr := range addrs { for _, addr := range addrs {
if addr.User == "" { if addr.User == "" {
// Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero. // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.
log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", mlog.Field("address", addr)) log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))
continue continue
} }
d, err := dns.ParseDomain(addr.Host) d, err := dns.ParseDomain(addr.Host)
if err != nil { if err != nil {
log.Debugx("parsing domain in to/cc/bcc address", err, mlog.Field("address", addr)) log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))
continue continue
} }
mr := Recipient{ mr := Recipient{
MessageID: m.ID, MessageID: m.ID,
Localpart: smtp.Localpart(addr.User), Localpart: smtp.Localpart(addr.User),
Domain: d.Name(), Domain: d.Name(),
OrgDomain: publicsuffix.Lookup(context.TODO(), d).Name(), OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),
Sent: sent, Sent: sent,
} }
if err := tx.Insert(&mr); err != nil { if err := tx.Insert(&mr); err != nil {
@ -1365,9 +1364,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
} }
if sync { if sync {
if err := moxio.SyncDir(msgDir); err != nil { if err := moxio.SyncDir(log, msgDir); err != nil {
xerr := os.Remove(msgPath) xerr := os.Remove(msgPath)
log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath)) log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
return fmt.Errorf("sync directory: %w", err) return fmt.Errorf("sync directory: %w", err)
} }
} }
@ -1376,7 +1375,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
l := []Message{*m} l := []Message{*m}
if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil { if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {
xerr := os.Remove(msgPath) xerr := os.Remove(msgPath)
log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath)) log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
return fmt.Errorf("training junkfilter: %w", err) return fmt.Errorf("training junkfilter: %w", err)
} }
*m = l[0] *m = l[0]
@ -1387,7 +1386,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
// SetPassword saves a new password for this account. This password is used for // SetPassword saves a new password for this account. This password is used for
// IMAP, SMTP (submission) sessions and the HTTP account web page. // IMAP, SMTP (submission) sessions and the HTTP account web page.
func (a *Account) SetPassword(password string) error { func (a *Account) SetPassword(log mlog.Log, password string) error {
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil { if err != nil {
return fmt.Errorf("generating password hash: %w", err) return fmt.Errorf("generating password hash: %w", err)
@ -1439,7 +1438,7 @@ func (a *Account) SetPassword(password string) error {
return nil return nil
}) })
if err == nil { if err == nil {
xlog.Info("new password set for account", mlog.Field("account", a.Name)) log.Info("new password set for account", slog.String("account", a.Name))
} }
return err return err
} }
@ -1590,21 +1589,21 @@ func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, erro
// MessageRuleset returns the first ruleset (if any) that message the message // MessageRuleset returns the first ruleset (if any) that message the message
// represented by msgPrefix and msgFile, with smtp and validation fields from m. // represented by msgPrefix and msgFile, with smtp and validation fields from m.
func MessageRuleset(log *mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset { func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {
if len(dest.Rulesets) == 0 { if len(dest.Rulesets) == 0 {
return nil return nil
} }
mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile. mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.
p, err := message.Parse(log, false, mr) p, err := message.Parse(log.Logger, false, mr)
if err != nil { if err != nil {
log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, mlog.Field("parse", "")) log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))
// note: part is still set. // note: part is still set.
} }
// todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing. // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.
header, err := p.Header() header, err := p.Header()
if err != nil { if err != nil {
log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, mlog.Field("parse", "")) log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))
// todo: reject message? // todo: reject message?
return nil return nil
} }
@ -1678,7 +1677,7 @@ func (a *Account) MessageReader(m Message) *MsgReader {
// Caller must hold account wlock (mailbox may be created). // Caller must hold account wlock (mailbox may be created).
// Message delivery, possible mailbox creation, and updated mailbox counts are // Message delivery, possible mailbox creation, and updated mailbox counts are
// broadcasted. // broadcasted.
func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error { func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {
var mailbox string var mailbox string
rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile) rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)
if rs != nil { if rs != nil {
@ -1696,7 +1695,7 @@ func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m *
// Caller must hold account wlock (mailbox may be created). // Caller must hold account wlock (mailbox may be created).
// Message delivery, possible mailbox creation, and updated mailbox counts are // Message delivery, possible mailbox creation, and updated mailbox counts are
// broadcasted. // broadcasted.
func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgFile *os.File) error { func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {
var changes []Change var changes []Change
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error { err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, chl, err := a.MailboxEnsure(tx, mailbox, true) mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
@ -1734,7 +1733,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF
// //
// Caller most hold account wlock. // Caller most hold account wlock.
// Changes are broadcasted. // Changes are broadcasted.
func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) { func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {
var changes []Change var changes []Change
var remove []Message var remove []Message
@ -1742,7 +1741,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
for _, m := range remove { for _, m := range remove {
p := a.MessagePath(m.ID) p := a.MessagePath(m.ID)
err := os.Remove(p) err := os.Remove(p)
log.Check(err, "removing rejects message file", mlog.Field("path", p)) log.Check(err, "removing rejects message file", slog.String("path", p))
} }
}() }()
@ -1796,7 +1795,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
return hasSpace, nil return hasSpace, nil
} }
func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) { func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
if len(l) == 0 { if len(l) == 0 {
return nil, nil return nil, nil
} }
@ -1858,7 +1857,7 @@ func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx *
// RejectsRemove removes a message from the rejects mailbox if present. // RejectsRemove removes a message from the rejects mailbox if present.
// Caller most hold account wlock. // Caller most hold account wlock.
// Changes are broadcasted. // Changes are broadcasted.
func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string) error { func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {
var changes []Change var changes []Change
var remove []Message var remove []Message
@ -1866,7 +1865,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string)
for _, m := range remove { for _, m := range remove {
p := a.MessagePath(m.ID) p := a.MessagePath(m.ID)
err := os.Remove(p) err := os.Remove(p)
log.Check(err, "removing rejects message file", mlog.Field("path", p)) log.Check(err, "removing rejects message file", slog.String("path", p))
} }
}() }()
@ -1933,8 +1932,8 @@ func manageAuthCache() {
// OpenEmailAuth opens an account given an email address and password. // OpenEmailAuth opens an account given an email address and password.
// //
// The email address may contain a catchall separator. // The email address may contain a catchall separator.
func OpenEmailAuth(email string, password string) (acc *Account, rerr error) { func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {
acc, _, rerr = OpenEmail(email) acc, _, rerr = OpenEmail(log, email)
if rerr != nil { if rerr != nil {
return return
} }
@ -1942,7 +1941,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
defer func() { defer func() {
if rerr != nil && acc != nil { if rerr != nil && acc != nil {
err := acc.Close() err := acc.Close()
xlog.Check(err, "closing account after open auth failure") log.Check(err, "closing account after open auth failure")
acc = nil acc = nil
} }
}() }()
@ -1973,7 +1972,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
// OpenEmail opens an account given an email address. // OpenEmail opens an account given an email address.
// //
// The email address may contain a catchall separator. // The email address may contain a catchall separator.
func OpenEmail(email string) (*Account, config.Destination, error) { func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {
addr, err := smtp.ParseAddress(email) addr, err := smtp.ParseAddress(email)
if err != nil { if err != nil {
return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err) return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)
@ -1984,7 +1983,7 @@ func OpenEmail(email string) (*Account, config.Destination, error) {
} else if err != nil { } else if err != nil {
return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err) return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)
} }
acc, err := OpenAccount(accountName) acc, err := OpenAccount(log, accountName)
if err != nil { if err != nil {
return nil, config.Destination{}, err return nil, config.Destination{}, err
} }
@ -2383,7 +2382,7 @@ func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (chang
// indicates that and an error is returned. // indicates that and an error is returned.
// //
// Caller should broadcast the changes and remove files for the removed message IDs. // Caller should broadcast the changes and remove files for the removed message IDs.
func (a *Account) MailboxDelete(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) { func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
// Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about
// NoInferior and NoSelect. We just require only leaf mailboxes are deleted. // NoInferior and NoSelect. We just require only leaf mailboxes are deleted.
qmb := bstore.QueryTx[Mailbox](tx) qmb := bstore.QueryTx[Mailbox](tx)

34
store/account_test.go
View file

@ -19,6 +19,7 @@ import (
) )
var ctxbg = context.Background() var ctxbg = context.Background()
var pkglog = mlog.New("store", nil)
func tcheck(t *testing.T, err error, msg string) { func tcheck(t *testing.T, err error, msg string) {
t.Helper() t.Helper()
@ -28,10 +29,11 @@ func tcheck(t *testing.T, err error, msg string) {
} }
func TestMailbox(t *testing.T) { func TestMailbox(t *testing.T) {
log := mlog.New("store", nil)
os.RemoveAll("../testdata/store/data") os.RemoveAll("../testdata/store/data")
mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf") mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")
mox.MustLoadConfig(true, false) mox.MustLoadConfig(true, false)
acc, err := OpenAccount("mjl") acc, err := OpenAccount(log, "mjl")
tcheck(t, err, "open account") tcheck(t, err, "open account")
defer func() { defer func() {
err = acc.Close() err = acc.Close()
@ -39,9 +41,7 @@ func TestMailbox(t *testing.T) {
}() }()
defer Switchboard()() defer Switchboard()()
log := mlog.New("store") msgFile, err := CreateMessageTemp(log, "account-test")
msgFile, err := CreateMessageTemp("account-test")
if err != nil { if err != nil {
t.Fatalf("creating temp msg file: %s", err) t.Fatalf("creating temp msg file: %s", err)
} }
@ -72,7 +72,7 @@ func TestMailbox(t *testing.T) {
} }
acc.WithWLock(func() { acc.WithWLock(func() {
conf, _ := acc.Conf() conf, _ := acc.Conf()
err := acc.DeliverDestination(xlog, conf.Destinations["mjl"], &m, msgFile) err := acc.DeliverDestination(log, conf.Destinations["mjl"], &m, msgFile)
tcheck(t, err, "deliver without consume") tcheck(t, err, "deliver without consume")
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error { err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
@ -81,7 +81,7 @@ func TestMailbox(t *testing.T) {
tcheck(t, err, "sent mailbox") tcheck(t, err, "sent mailbox")
msent.MailboxID = mbsent.ID msent.MailboxID = mbsent.ID
msent.MailboxOrigID = mbsent.ID msent.MailboxOrigID = mbsent.ID
err = acc.DeliverMessage(xlog, tx, &msent, msgFile, true, false, false) err = acc.DeliverMessage(pkglog, tx, &msent, msgFile, true, false, false)
tcheck(t, err, "deliver message") tcheck(t, err, "deliver message")
if !msent.ThreadMuted || !msent.ThreadCollapsed { if !msent.ThreadMuted || !msent.ThreadCollapsed {
t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m") t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m")
@ -97,7 +97,7 @@ func TestMailbox(t *testing.T) {
tcheck(t, err, "insert rejects mailbox") tcheck(t, err, "insert rejects mailbox")
mreject.MailboxID = mbrejects.ID mreject.MailboxID = mbrejects.ID
mreject.MailboxOrigID = mbrejects.ID mreject.MailboxOrigID = mbrejects.ID
err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, true, false, false) err = acc.DeliverMessage(pkglog, tx, &mreject, msgFile, true, false, false)
tcheck(t, err, "deliver message") tcheck(t, err, "deliver message")
err = tx.Get(&mbrejects) err = tx.Get(&mbrejects)
@ -110,7 +110,7 @@ func TestMailbox(t *testing.T) {
}) })
tcheck(t, err, "deliver as sent and rejects") tcheck(t, err, "deliver as sent and rejects")
err = acc.DeliverDestination(xlog, conf.Destinations["mjl"], &mconsumed, msgFile) err = acc.DeliverDestination(pkglog, conf.Destinations["mjl"], &mconsumed, msgFile)
tcheck(t, err, "deliver with consume") tcheck(t, err, "deliver with consume")
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error { err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
@ -141,7 +141,7 @@ func TestMailbox(t *testing.T) {
}) })
tcheck(t, err, "untraining non-junk") tcheck(t, err, "untraining non-junk")
err = acc.SetPassword("testtest") err = acc.SetPassword(log, "testtest")
tcheck(t, err, "set password") tcheck(t, err, "set password")
key0, err := acc.Subjectpass("test@localhost") key0, err := acc.Subjectpass("test@localhost")
@ -223,37 +223,37 @@ func TestMailbox(t *testing.T) {
// Run the auth tests twice for possible cache effects. // Run the auth tests twice for possible cache effects.
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
_, err := OpenEmailAuth("mjl@mox.example", "bogus") _, err := OpenEmailAuth(log, "mjl@mox.example", "bogus")
if err != ErrUnknownCredentials { if err != ErrUnknownCredentials {
t.Fatalf("got %v, expected ErrUnknownCredentials", err) t.Fatalf("got %v, expected ErrUnknownCredentials", err)
} }
} }
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
acc2, err := OpenEmailAuth("mjl@mox.example", "testtest") acc2, err := OpenEmailAuth(log, "mjl@mox.example", "testtest")
tcheck(t, err, "open for email with auth") tcheck(t, err, "open for email with auth")
err = acc2.Close() err = acc2.Close()
tcheck(t, err, "close account") tcheck(t, err, "close account")
} }
acc2, err := OpenEmailAuth("other@mox.example", "testtest") acc2, err := OpenEmailAuth(log, "other@mox.example", "testtest")
tcheck(t, err, "open for email with auth") tcheck(t, err, "open for email with auth")
err = acc2.Close() err = acc2.Close()
tcheck(t, err, "close account") tcheck(t, err, "close account")
_, err = OpenEmailAuth("bogus@mox.example", "testtest") _, err = OpenEmailAuth(log, "bogus@mox.example", "testtest")
if err != ErrUnknownCredentials { if err != ErrUnknownCredentials {
t.Fatalf("got %v, expected ErrUnknownCredentials", err) t.Fatalf("got %v, expected ErrUnknownCredentials", err)
} }
_, err = OpenEmailAuth("mjl@test.example", "testtest") _, err = OpenEmailAuth(log, "mjl@test.example", "testtest")
if err != ErrUnknownCredentials { if err != ErrUnknownCredentials {
t.Fatalf("got %v, expected ErrUnknownCredentials", err) t.Fatalf("got %v, expected ErrUnknownCredentials", err)
} }
} }
func TestMessageRuleset(t *testing.T) { func TestMessageRuleset(t *testing.T) {
f, err := CreateMessageTemp("msgruleset") f, err := CreateMessageTemp(pkglog, "msgruleset")
tcheck(t, err, "creating temp msg file") tcheck(t, err, "creating temp msg file")
defer os.Remove(f.Name()) defer os.Remove(f.Name())
defer f.Close() defer f.Close()
@ -284,7 +284,7 @@ Rulesets:
} }
dest.Rulesets[0].HeadersRegexpCompiled = hdrs dest.Rulesets[0].HeadersRegexpCompiled = hdrs
c := MessageRuleset(xlog, dest, &Message{}, msgBuf, f) c := MessageRuleset(pkglog, dest, &Message{}, msgBuf, f)
if c == nil { if c == nil {
t.Fatalf("expected ruleset match") t.Fatalf("expected ruleset match")
} }
@ -293,7 +293,7 @@ Rulesets:
test test
`, "\n", "\r\n")) `, "\n", "\r\n"))
c = MessageRuleset(xlog, dest, &Message{}, msg2Buf, f) c = MessageRuleset(pkglog, dest, &Message{}, msg2Buf, f)
if c != nil { if c != nil {
t.Fatalf("expected no ruleset match") t.Fatalf("expected no ruleset match")
} }

8
store/cleanuptemp.go
View file

@ -3,15 +3,17 @@ package store
import ( import (
"os" "os"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
// CloseRemoveTempFile closes and removes f, a file described by descr. Often // CloseRemoveTempFile closes and removes f, a file described by descr. Often
// used in a defer after creating a temporary file. // used in a defer after creating a temporary file.
func CloseRemoveTempFile(log *mlog.Log, f *os.File, descr string) { func CloseRemoveTempFile(log mlog.Log, f *os.File, descr string) {
name := f.Name() name := f.Name()
err := f.Close() err := f.Close()
log.Check(err, "closing temporary file", mlog.Field("kind", descr)) log.Check(err, "closing temporary file", slog.String("kind", descr))
err = os.Remove(name) err = os.Remove(name)
log.Check(err, "removing temporary file", mlog.Field("kind", descr)) log.Check(err, "removing temporary file", slog.String("kind", descr))
} }

Some files were not shown because too many files have changed in this diff Show more