serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-25 16:03:48 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

switch to slog.Logger for logging, for easier reuse of packages by external software

Browse source 
we don't want external software to include internal details like mlog.
slog.Logger is/will be the standard.

we still have mlog for its helper functions, and its handler that logs in
concise logfmt used by mox.

packages that are not meant for reuse still pass around mlog.Log for
convenience.

we use golang.org/x/exp/slog because we also support the previous Go toolchain
version. with the next Go release, we'll switch to the builtin slog.
This commit is contained in:
Mechiel Lukkien 2023-12-05 13:35:58 +01:00
parent 56b2a9d980
commit 5b20cba50a
No known key found for this signature in database
150 changed files with 5176 additions and 1898 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
autotls/autotls.go
View file

@ -28,9 +28,11 @@ import (
"sync"
"time"
"golang.org/x/crypto/acme"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"golang.org/x/crypto/acme"
"github.com/mjl-/autocert"
@ -39,8 +41,6 @@ import (
"github.com/mjl-/mox/moxvar"
)
var xlog = mlog.New("autotls")
var (
metricCertput = promauto.NewCounter(
prometheus.CounterOpts{
@ -148,7 +148,7 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h
}
loggingGetCertificate := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
log := xlog.WithContext(hello.Context())
log := mlog.New("autotls", nil).WithContext(hello.Context())
// Handle missing SNI to prevent logging an error below.
// At startup, during config initialization, we already adjust the tls config to
@ -156,16 +156,16 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h
// common for SMTP STARTTLS connections, which often do not care about the
// verification of the certificate.
if hello.ServerName == "" {
log.Debug("tls request without sni servername, rejecting", mlog.Field("localaddr", hello.Conn.LocalAddr()), mlog.Field("supportedprotos", hello.SupportedProtos))
log.Debug("tls request without sni servername, rejecting", slog.Any("localaddr", hello.Conn.LocalAddr()), slog.Any("supportedprotos", hello.SupportedProtos))
return nil, fmt.Errorf("sni server name required")
}
cert, err := m.GetCertificate(hello)
if err != nil {
if errors.Is(err, errHostNotAllowed) {
log.Debugx("requesting certificate", err, mlog.Field("host", hello.ServerName))
log.Debugx("requesting certificate", err, slog.String("host", hello.ServerName))
} else {
log.Errorx("requesting certificate", err, mlog.Field("host", hello.ServerName))
log.Errorx("requesting certificate", err, slog.String("host", hello.ServerName))
}
}
return cert, err
@ -194,7 +194,7 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h
// are fully served by publicIPs (only if non-empty and there is no unspecified
// address in the list). If no, log an error with a warning that ACME validation
// may fail.
func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.Domain]struct{}, publicIPs []string, checkHosts bool) {
func (m *Manager) SetAllowedHostnames(log mlog.Log, resolver dns.Resolver, hostnames map[dns.Domain]struct{}, publicIPs []string, checkHosts bool) {
m.Lock()
defer m.Unlock()
@ -207,7 +207,7 @@ func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.D
return l[i].Name() < l[j].Name()
})
xlog.Debug("autotls setting allowed hostnames", mlog.Field("hostnames", l), mlog.Field("publicips", publicIPs))
log.Debug("autotls setting allowed hostnames", slog.Any("hostnames", l), slog.Any("publicips", publicIPs))
var added []dns.Domain
for h := range hostnames {
if _, ok := m.hosts[h]; !ok {
@ -231,16 +231,16 @@ func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.D
publicIPstrs[ip] = struct{}{}
}
xlog.Debug("checking ips of hosts configured for acme tls cert validation")
log.Debug("checking ips of hosts configured for acme tls cert validation")
for _, h := range added {
ips, _, err := resolver.LookupIP(ctx, "ip", h.ASCII+".")
if err != nil {
xlog.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, mlog.Field("host", h))
log.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, slog.Any("host", h))
continue
}
for _, ip := range ips {
if _, ok := publicIPstrs[ip.String()]; !ok {
xlog.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on", mlog.Field("hostname", h), mlog.Field("listenedips", publicIPs), mlog.Field("hostips", ips), mlog.Field("missingip", ip))
log.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on", slog.Any("hostname", h), slog.Any("listenedips", publicIPs), slog.Any("hostips", ips), slog.Any("missingip", ip))
}
}
}
@ -266,9 +266,9 @@ var errHostNotAllowed = errors.New("autotls: host not in allowlist")
// present. Only hosts added with SetAllowedHostnames are allowed. During shutdown,
// no new connections are allowed.
func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) {
log := xlog.WithContext(ctx)
log := mlog.New("autotls", nil).WithContext(ctx)
defer func() {
log.WithContext(ctx).Debugx("autotls hostpolicy result", rerr, mlog.Field("host", host))
log.Debugx("autotls hostpolicy result", rerr, slog.String("host", host))
}()
// Don't request new TLS certs when we are shutting down.
@ -300,46 +300,46 @@ func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) {
type dirCache autocert.DirCache
func (d dirCache) Delete(ctx context.Context, name string) (rerr error) {
log := xlog.WithContext(ctx)
log := mlog.New("autotls", nil).WithContext(ctx)
defer func() {
log.Debugx("dircache delete result", rerr, mlog.Field("name", name))
log.Debugx("dircache delete result", rerr, slog.String("name", name))
}()
err := autocert.DirCache(d).Delete(ctx, name)
if err != nil {
log.Errorx("deleting cert from dir cache", err, mlog.Field("name", name))
log.Errorx("deleting cert from dir cache", err, slog.String("name", name))
} else if !strings.HasSuffix(name, "+token") {
log.Info("autotls cert delete", mlog.Field("name", name))
log.Info("autotls cert delete", slog.String("name", name))
}
return err
}
func (d dirCache) Get(ctx context.Context, name string) (rbuf []byte, rerr error) {
log := xlog.WithContext(ctx)
log := mlog.New("autotls", nil).WithContext(ctx)
defer func() {
log.Debugx("dircache get result", rerr, mlog.Field("name", name))
log.Debugx("dircache get result", rerr, slog.String("name", name))
}()
buf, err := autocert.DirCache(d).Get(ctx, name)
if err != nil && errors.Is(err, autocert.ErrCacheMiss) {
log.Infox("getting cert from dir cache", err, mlog.Field("name", name))
log.Infox("getting cert from dir cache", err, slog.String("name", name))
} else if err != nil {
log.Errorx("getting cert from dir cache", err, mlog.Field("name", name))
log.Errorx("getting cert from dir cache", err, slog.String("name", name))
} else if !strings.HasSuffix(name, "+token") {
log.Debug("autotls cert get", mlog.Field("name", name))
log.Debug("autotls cert get", slog.String("name", name))
}
return buf, err
}
func (d dirCache) Put(ctx context.Context, name string, data []byte) (rerr error) {
log := xlog.WithContext(ctx)
log := mlog.New("autotls", nil).WithContext(ctx)
defer func() {
log.Debugx("dircache put result", rerr, mlog.Field("name", name))
log.Debugx("dircache put result", rerr, slog.String("name", name))
}()
metricCertput.Inc()
err := autocert.DirCache(d).Put(ctx, name, data)
if err != nil {
log.Errorx("storing cert in dir cache", err, mlog.Field("name", name))
log.Errorx("storing cert in dir cache", err, slog.String("name", name))
} else if !strings.HasSuffix(name, "+token") {
log.Info("autotls cert store", mlog.Field("name", name))
log.Info("autotls cert store", slog.String("name", name))
}
return err
}

6
autotls/autotls_test.go
View file

@ -12,9 +12,11 @@ import (
"github.com/mjl-/autocert"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
)
func TestAutotls(t *testing.T) {
log := mlog.New("autotls", nil)
os.RemoveAll("../testdata/autotls")
os.MkdirAll("../testdata/autotls", 0770)
@ -34,7 +36,7 @@ func TestAutotls(t *testing.T) {
if err := m.HostPolicy(context.Background(), "mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) {
t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err)
}
m.SetAllowedHostnames(dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false)
m.SetAllowedHostnames(log, dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false)
l = m.Hostnames()
if !reflect.DeepEqual(l, []dns.Domain{{ASCII: "mox.example"}}) {
t.Fatalf("hostnames, got %v, expected single mox.example", l)
@ -88,7 +90,7 @@ func TestAutotls(t *testing.T) {
t.Fatalf("private key changed after reload")
}
m.shutdown = make(chan struct{})
m.SetAllowedHostnames(dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false)
m.SetAllowedHostnames(log, dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false)
if err := m.HostPolicy(context.Background(), "mox.example"); err != nil {
t.Fatalf("hostpolicy, got err %v, expected no error", err)
}

111
backup.go
View file

@ -12,10 +12,11 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/mtastsdb"
@ -48,51 +49,51 @@ func backupctl(ctx context.Context, ctl *ctl) {
writer := ctl.writer()
// Format easily readable output for the user.
formatLog := func(prefix, text string, err error, fields ...mlog.Pair) []byte {
formatLog := func(prefix, text string, err error, attrs ...slog.Attr) []byte {
var b bytes.Buffer
fmt.Fprint(&b, prefix)
fmt.Fprint(&b, text)
if err != nil {
fmt.Fprint(&b, ": "+err.Error())
}
for _, f := range fields {
fmt.Fprintf(&b, "; %s=%v", f.Key, f.Value)
for _, a := range attrs {
fmt.Fprintf(&b, "; %s=%v", a.Key, a.Value)
}
fmt.Fprint(&b, "\n")
return b.Bytes()
}
// Log an error to both the mox service as the user running "mox backup".
xlogx := func(prefix, text string, err error, fields ...mlog.Pair) {
ctl.log.Errorx(text, err, fields...)
pkglogx := func(prefix, text string, err error, attrs ...slog.Attr) {
ctl.log.Errorx(text, err, attrs...)
_, werr := writer.Write(formatLog(prefix, text, err, fields...))
_, werr := writer.Write(formatLog(prefix, text, err, attrs...))
ctl.xcheck(werr, "write to ctl")
}
// Log an error but don't mark backup as failed.
xwarnx := func(text string, err error, fields ...mlog.Pair) {
xlogx("warning: ", text, err, fields...)
xwarnx := func(text string, err error, attrs ...slog.Attr) {
pkglogx("warning: ", text, err, attrs...)
}
// Log an error that causes the backup to be marked as failed. We typically
// continue processing though.
xerrx := func(text string, err error, fields ...mlog.Pair) {
xerrx := func(text string, err error, attrs ...slog.Attr) {
incomplete = true
xlogx("error: ", text, err, fields...)
pkglogx("error: ", text, err, attrs...)
}
// If verbose is enabled, log to the cli command. Always log as info level.
xvlog := func(text string, fields ...mlog.Pair) {
ctl.log.Info(text, fields...)
xvlog := func(text string, attrs ...slog.Attr) {
ctl.log.Info(text, attrs...)
if verbose {
_, werr := writer.Write(formatLog("", text, nil, fields...))
_, werr := writer.Write(formatLog("", text, nil, attrs...))
ctl.xcheck(werr, "write to ctl")
}
}
if _, err := os.Stat(dstDataDir); err == nil {
xwarnx("destination data directory already exists", nil, mlog.Field("dir", dstDataDir))
xwarnx("destination data directory already exists", nil, slog.String("dir", dstDataDir))
}
srcDataDir := filepath.Clean(mox.DataDirPath("."))
@ -119,7 +120,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
sf, err := os.Open(srcpath)
if err != nil {
xerrx("open source file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xerrx("open source file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return
}
defer sf.Close()
@ -127,7 +128,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
ensureDestDir(dstpath)
df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
if err != nil {
xerrx("creating destination file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xerrx("creating destination file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return
}
defer func() {
@ -136,16 +137,16 @@ func backupctl(ctx context.Context, ctl *ctl) {
}
}()
if _, err := io.Copy(df, sf); err != nil {
xerrx("copying file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xerrx("copying file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return
}
err = df.Close()
df = nil
if err != nil {
xerrx("closing destination file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xerrx("closing destination file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
return
}
xvlog("backed up file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmFile)))
xvlog("backed up file", slog.String("path", path), slog.Duration("duration", time.Since(tmFile)))
}
// Back up the files in a directory (by copying).
@ -155,7 +156,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
dstdir := filepath.Join(dstDataDir, dir)
err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking file (not backed up)", err, mlog.Field("srcpath", srcpath))
xerrx("walking file (not backed up)", err, slog.String("srcpath", srcpath))
return nil
}
if d.IsDir() {
@ -165,10 +166,10 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil
})
if err != nil {
xerrx("copying directory (not backed up properly)", err, mlog.Field("srcdir", srcdir), mlog.Field("dstdir", dstdir), mlog.Field("duration", time.Since(tmDir)))
xerrx("copying directory (not backed up properly)", err, slog.String("srcdir", srcdir), slog.String("dstdir", dstdir), slog.Duration("duration", time.Since(tmDir)))
return
}
xvlog("backed up directory", mlog.Field("dir", dir), mlog.Field("duration", time.Since(tmDir)))
xvlog("backed up directory", slog.String("dir", dir), slog.Duration("duration", time.Since(tmDir)))
}
// Backup a database by copying it in a readonly transaction.
@ -177,7 +178,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
backupDB := func(db *bstore.DB, path string) (rerr error) {
defer func() {
if rerr != nil {
xerrx("backing up database", rerr, mlog.Field("path", path))
xerrx("backing up database", rerr, slog.String("path", path))
}
}()
@ -216,7 +217,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
if err != nil {
return fmt.Errorf("closing destination database after copy: %v", err)
}
xvlog("backed up database file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmDB)))
xvlog("backed up database file", slog.String("path", path), slog.Duration("duration", time.Since(tmDB)))
return nil
}
@ -231,7 +232,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
// No point in trying with regular copy, we would warn twice.
return false, err
} else if !warnedHardlink {
xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
warnedHardlink = true
}
@ -269,7 +270,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
// Start making the backup.
tmStart := time.Now()
ctl.log.Print("making backup", mlog.Field("destdir", dstDataDir))
ctl.log.Print("making backup", slog.String("destdir", dstDataDir))
err := os.MkdirAll(dstDataDir, 0770)
if err != nil {
@ -299,14 +300,14 @@ func backupctl(ctx context.Context, ctl *ctl) {
tmQueue := time.Now()
if err := backupDB(queue.DB, path); err != nil {
xerrx("queue not backed up", err, mlog.Field("path", path), mlog.Field("duration", time.Since(tmQueue)))
xerrx("queue not backed up", err, slog.String("path", path), slog.Duration("duration", time.Since(tmQueue)))
return
}
dstdbpath := filepath.Join(dstDataDir, path)
db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)
if err != nil {
xerrx("open copied queue database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmQueue)))
xerrx("open copied queue database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmQueue)))
return
}
@ -329,7 +330,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcpath := filepath.Join(srcDataDir, "queue", mp)
dstpath := filepath.Join(dstDataDir, "queue", mp)
if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
xerrx("linking/copying queue message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xerrx("linking/copying queue message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
} else if linked {
nlinked++
} else {
@ -338,9 +339,9 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil
})
if err != nil {
xerrx("processing queue messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs)))
xerrx("processing queue messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs)))
} else {
xvlog("queue message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs)))
xvlog("queue message files linked/copied", slog.Int("linked", nlinked), slog.Int("copied", ncopied), slog.Duration("duration", time.Since(tmMsgs)))
}
// Read through all files in queue directory and warn about anything we haven't handled yet.
@ -348,7 +349,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcqdir := filepath.Join(srcDataDir, "queue")
err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking files in queue", err, mlog.Field("srcpath", srcqpath))
xerrx("walking files in queue", err, slog.String("srcpath", srcqpath))
return nil
}
if d.IsDir() {
@ -362,17 +363,17 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil
}
qp := filepath.Join("queue", p)
xwarnx("backing up unrecognized file in queue directory", nil, mlog.Field("path", qp))
xwarnx("backing up unrecognized file in queue directory", nil, slog.String("path", qp))
backupFile(qp)
return nil
})
if err != nil {
xerrx("walking queue directory (not backed up properly)", err, mlog.Field("dir", "queue"), mlog.Field("duration", time.Since(tmWalk)))
xerrx("walking queue directory (not backed up properly)", err, slog.String("dir", "queue"), slog.Duration("duration", time.Since(tmWalk)))
} else {
xvlog("walked queue directory", mlog.Field("duration", time.Since(tmWalk)))
xvlog("walked queue directory", slog.Duration("duration", time.Since(tmWalk)))
}
xvlog("queue backed finished", mlog.Field("duration", time.Since(tmQueue)))
xvlog("queue backed finished", slog.Duration("duration", time.Since(tmQueue)))
}
backupQueue(filepath.FromSlash("queue/index.db"))
@ -385,7 +386,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
dbpath := filepath.Join("accounts", acc.Name, "index.db")
err := backupDB(acc.DB, dbpath)
if err != nil {
xerrx("copying account database", err, mlog.Field("path", dbpath), mlog.Field("duration", time.Since(tmAccount)))
xerrx("copying account database", err, slog.String("path", dbpath), slog.Duration("duration", time.Since(tmAccount)))
}
// todo: should document/check not taking a rlock on account.
@ -409,7 +410,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
dstdbpath := filepath.Join(dstDataDir, dbpath)
db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...)
if err != nil {
xerrx("open copied account database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmAccount)))
xerrx("open copied account database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmAccount)))
return
}
@ -433,7 +434,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcpath := filepath.Join(srcDataDir, amp)
dstpath := filepath.Join(dstDataDir, amp)
if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
xerrx("linking/copying account message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
xerrx("linking/copying account message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath))
} else if linked {
nlinked++
} else {
@ -442,9 +443,9 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil
})
if err != nil {
xerrx("processing account messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs)))
xerrx("processing account messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs)))
} else {
xvlog("account message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs)))
xvlog("account message files linked/copied", slog.Int("linked", nlinked), slog.Int("copied", ncopied), slog.Duration("duration", time.Since(tmMsgs)))
}
// Read through all files in account directory and warn about anything we haven't handled yet.
@ -452,7 +453,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
srcadir := filepath.Join(srcDataDir, "accounts", acc.Name)
err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking files in account", err, mlog.Field("srcpath", srcapath))
xerrx("walking files in account", err, slog.String("srcpath", srcapath))
return nil
}
if d.IsDir() {
@ -472,20 +473,20 @@ func backupctl(ctx context.Context, ctl *ctl) {
}
ap := filepath.Join("accounts", acc.Name, p)
if strings.HasPrefix(p, "msg"+string(filepath.Separator)) {
xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, mlog.Field("path", ap))
xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, slog.String("path", ap))
} else {
xwarnx("backing up unrecognized file in account directory", nil, mlog.Field("path", ap))
xwarnx("backing up unrecognized file in account directory", nil, slog.String("path", ap))
}
backupFile(ap)
return nil
})
if err != nil {
xerrx("walking account directory (not backed up properly)", err, mlog.Field("srcdir", srcadir), mlog.Field("duration", time.Since(tmWalk)))
xerrx("walking account directory (not backed up properly)", err, slog.String("srcdir", srcadir), slog.Duration("duration", time.Since(tmWalk)))
} else {
xvlog("walked account directory", mlog.Field("duration", time.Since(tmWalk)))
xvlog("walked account directory", slog.Duration("duration", time.Since(tmWalk)))
}
xvlog("account backup finished", mlog.Field("dir", filepath.Join("accounts", acc.Name)), mlog.Field("duration", time.Since(tmAccount)))
xvlog("account backup finished", slog.String("dir", filepath.Join("accounts", acc.Name)), slog.Duration("duration", time.Since(tmAccount)))
}
// For each configured account, open it, make a copy of the database and
@ -493,9 +494,9 @@ func backupctl(ctx context.Context, ctl *ctl) {
// account directories when handling "all other files" below.
accounts := map[string]struct{}{}
for _, accName := range mox.Conf.Accounts() {
acc, err := store.OpenAccount(accName)
acc, err := store.OpenAccount(ctl.log, accName)
if err != nil {
xerrx("opening account for copying (will try to copy as regular files later)", err, mlog.Field("account", accName))
xerrx("opening account for copying (will try to copy as regular files later)", err, slog.String("account", accName))
continue
}
accounts[accName] = struct{}{}
@ -506,7 +507,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
tmWalk := time.Now()
err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking path", err, mlog.Field("path", srcpath))
xerrx("walking path", err, slog.String("path", srcpath))
return nil
}
@ -536,18 +537,18 @@ func backupctl(ctx context.Context, ctl *ctl) {
return nil
case "lastknownversion": // Optional file, not yet handled.
default:
xwarnx("backing up unrecognized file", nil, mlog.Field("path", p))
xwarnx("backing up unrecognized file", nil, slog.String("path", p))
}
backupFile(p)
return nil
})
if err != nil {
xerrx("walking other files (not backed up properly)", err, mlog.Field("duration", time.Since(tmWalk)))
xerrx("walking other files (not backed up properly)", err, slog.Duration("duration", time.Since(tmWalk)))
} else {
xvlog("walking other files finished", mlog.Field("duration", time.Since(tmWalk)))
xvlog("walking other files finished", slog.Duration("duration", time.Since(tmWalk)))
}
xvlog("backup finished", mlog.Field("duration", time.Since(tmStart)))
xvlog("backup finished", slog.Duration("duration", time.Since(tmStart)))
writer.xclose()

64
ctl.go
View file

@ -16,6 +16,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
@ -35,7 +37,7 @@ type ctl struct {
conn net.Conn
r *bufio.Reader // Set for first reader.
x any // If set, errors are handled by calling panic(x) instead of log.Fatal.
log *mlog.Log // If set, along with x, logging is done here.
log mlog.Log // If set, along with x, logging is done here.
}
// xctl opens a ctl connection.
@ -59,7 +61,7 @@ func (c *ctl) xerror(msg string) {
if c.x == nil {
log.Fatalln(msg)
}
c.log.Debugx("ctl error", fmt.Errorf("%s", msg), mlog.Field("cmd", c.cmd))
c.log.Debugx("ctl error", fmt.Errorf("%s", msg), slog.String("cmd", c.cmd))
c.xwrite(msg)
panic(c.x)
}
@ -74,7 +76,7 @@ func (c *ctl) xcheck(err error, msg string) {
if c.x == nil {
log.Fatalf("%s: %s", msg, err)
}
c.log.Debugx(msg, err, mlog.Field("cmd", c.cmd))
c.log.Debugx(msg, err, slog.String("cmd", c.cmd))
fmt.Fprintf(c.conn, "%s: %s\n", msg, err)
panic(c.x)
}
@ -160,7 +162,7 @@ type ctlwriter struct {
conn net.Conn // Ctl socket from which messages are read.
buf []byte // Scratch buffer, for reading response.
x any // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal.
log *mlog.Log
log mlog.Log
}
func (s *ctlwriter) Write(buf []byte) (int, error) {
@ -184,7 +186,7 @@ func (s *ctlwriter) xerror(msg string) {
if s.x == nil {
log.Fatalln(msg)
} else {
s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd))
s.log.Debugx("error", fmt.Errorf("%s", msg), slog.String("cmd", s.cmd))
panic(s.x)
}
}
@ -196,7 +198,7 @@ func (s *ctlwriter) xcheck(err error, msg string) {
if s.x == nil {
log.Fatalf("%s: %s", msg, err)
} else {
s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd))
s.log.Debugx(msg, err, slog.String("cmd", s.cmd))
panic(s.x)
}
}
@ -213,7 +215,7 @@ type ctlreader struct {
err error // If set, returned for each read. can also be io.EOF.
npending int // Number of bytes that can still be read until a new count line must be read.
x any // If set, errors are handled with panic(x) instead of log.Fatal.
log *mlog.Log // If x is set, logging goes to log.
log mlog.Log // If x is set, logging goes to log.
}
func (s *ctlreader) Read(buf []byte) (N int, Err error) {
@ -252,7 +254,7 @@ func (s *ctlreader) xerror(msg string) {
if s.x == nil {
log.Fatalln(msg)
} else {
s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd))
s.log.Debugx("error", fmt.Errorf("%s", msg), slog.String("cmd", s.cmd))
panic(s.x)
}
}
@ -264,13 +266,13 @@ func (s *ctlreader) xcheck(err error, msg string) {
if s.x == nil {
log.Fatalf("%s: %s", msg, err)
} else {
s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd))
s.log.Debugx(msg, err, slog.String("cmd", s.cmd))
panic(s.x)
}
}
// servectl handles requests on the unix domain socket "ctl", e.g. for graceful shutdown, local mail delivery.
func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) {
func servectl(ctx context.Context, log mlog.Log, conn net.Conn, shutdown func()) {
log.Debug("ctl connection")
var stop = struct{}{} // Sentinel value for panic and recover.
@ -280,7 +282,7 @@ func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()
if x == nil || x == stop {
return
}
log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", ctl.cmd))
log.Error("servectl panic", slog.Any("err", x), slog.String("cmd", ctl.cmd))
debug.PrintStack()
metrics.PanicInc(metrics.Ctl)
}()
@ -297,7 +299,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
log := ctl.log
cmd := ctl.xread()
ctl.cmd = cmd
log.Info("ctl command", mlog.Field("cmd", cmd))
log.Info("ctl command", slog.String("cmd", cmd))
switch cmd {
case "stop":
shutdown()
@ -314,10 +316,10 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
*/
to := ctl.xread()
a, addr, err := store.OpenEmail(to)
a, addr, err := store.OpenEmail(ctl.log, to)
ctl.xcheck(err, "lookup destination address")
msgFile, err := store.CreateMessageTemp("ctl-deliver")
msgFile, err := store.CreateMessageTemp(ctl.log, "ctl-deliver")
ctl.xcheck(err, "creating temporary message file")
defer store.CloseRemoveTempFile(log, msgFile, "deliver message")
mw := message.NewWriter(msgFile)
@ -335,7 +337,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
a.WithWLock(func() {
err := a.DeliverDestination(log, addr, m, msgFile)
ctl.xcheck(err, "delivering message")
log.Info("message delivered through ctl", mlog.Field("to", to))
log.Info("message delivered through ctl", slog.Any("to", to))
})
err = a.Close()
@ -353,7 +355,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
account := ctl.xread()
pw := ctl.xread()
acc, err := store.OpenAccount(account)
acc, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "open account")
defer func() {
if acc != nil {
@ -362,7 +364,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
}
}()
err = acc.SetPassword(pw)
err = acc.SetPassword(ctl.log, pw)
ctl.xcheck(err, "setting password")
err = acc.Close()
ctl.xcheck(err, "closing account")
@ -442,7 +444,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
ctl.xcheck(err, "parsing id")
}
count, err := queue.Drop(ctx, id, todomain, recipient)
count, err := queue.Drop(ctx, ctl.log, id, todomain, recipient)
ctl.xcheck(err, "dropping messages from queue")
ctl.xwrite(fmt.Sprintf("%d", count))
ctl.xwriteok()
@ -586,13 +588,13 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
pkg := ctl.xread()
levelstr := ctl.xread()
if levelstr == "" {
mox.Conf.LogLevelRemove(pkg)
mox.Conf.LogLevelRemove(ctl.log, pkg)
} else {
level, ok := mlog.Levels[levelstr]
if !ok {
ctl.xerror("bad level")
}
mox.Conf.LogLevelSet(pkg, level)
mox.Conf.LogLevelSet(ctl.log, pkg, level)
}
ctl.xwriteok()
@ -603,7 +605,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
< "ok" or error
*/
account := ctl.xread()
acc, err := store.OpenAccount(account)
acc, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "open account")
defer func() {
if acc != nil {
@ -623,9 +625,9 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
dbPath := filepath.Join(basePath, acc.Name, "junkfilter.db")
bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom")
err := os.Remove(dbPath)
log.Check(err, "removing old junkfilter database file", mlog.Field("path", dbPath))
log.Check(err, "removing old junkfilter database file", slog.String("path", dbPath))
err = os.Remove(bloomPath)
log.Check(err, "removing old junkfilter bloom filter file", mlog.Field("path", bloomPath))
log.Check(err, "removing old junkfilter bloom filter file", slog.String("path", bloomPath))
// Open junk filter, this creates new files.
jf, _, err := acc.OpenJunkFilter(ctx, ctl.log)
@ -651,7 +653,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
return err
})
ctl.xcheck(err, "training messages")
ctl.log.Info("retrained messages", mlog.Field("total", total), mlog.Field("trained", trained))
ctl.log.Info("retrained messages", slog.Int("total", total), slog.Int("trained", trained))
// Close junk filter, marking success.
err = jf.Close()
@ -668,7 +670,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
< stream
*/
account := ctl.xread()
acc, err := store.OpenAccount(account)
acc, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "open account")
defer func() {
if acc != nil {
@ -724,7 +726,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
const batchSize = 10000
xfixmsgsize := func(accName string) {
acc, err := store.OpenAccount(accName)
acc, err := store.OpenAccount(ctl.log, accName)
ctl.xcheck(err, "open account")
defer func() {
err := acc.Close()
@ -791,7 +793,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
m.Size = correctSize
mr := acc.MessageReader(m)
part, err := message.EnsurePart(log, false, mr, m.Size)
part, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil {
_, werr := fmt.Fprintf(w, "parsing message %d again: %v (continuing)\n", m.ID, err)
ctl.xcheck(werr, "write")
@ -859,7 +861,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
const batchSize = 100
xreparseAccount := func(accName string) {
acc, err := store.OpenAccount(accName)
acc, err := store.OpenAccount(ctl.log, accName)
ctl.xcheck(err, "open account")
defer func() {
err := acc.Close()
@ -880,7 +882,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
return q.ForEach(func(m store.Message) error {
lastID = m.ID
mr := acc.MessageReader(m)
p, err := message.EnsurePart(log, false, mr, m.Size)
p, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil {
_, err := fmt.Fprintf(w, "parsing message %d: %v (continuing)\n", m.ID, err)
ctl.xcheck(err, "write")
@ -935,7 +937,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
w := ctl.writer()
xreassignThreads := func(accName string) {
acc, err := store.OpenAccount(accName)
acc, err := store.OpenAccount(ctl.log, accName)
ctl.xcheck(err, "open account")
defer func() {
err := acc.Close()
@ -982,7 +984,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
backupctl(ctx, ctl)
default:
log.Info("unrecognized command", mlog.Field("cmd", cmd))
log.Info("unrecognized command", slog.String("cmd", cmd))
ctl.xwrite("unrecognized command")
return
}

19
ctl_test.go
View file

@ -21,6 +21,7 @@ import (
)
var ctxbg = context.Background()
var pkglog = mlog.New("ctl", nil)
func tcheck(t *testing.T, err error, errmsg string) {
if err != nil {
@ -36,19 +37,17 @@ func TestCtl(t *testing.T) {
os.RemoveAll("testdata/ctl/data")
mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf")
mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf")
if errs := mox.LoadConfig(ctxbg, true, false); len(errs) > 0 {
if errs := mox.LoadConfig(ctxbg, pkglog, true, false); len(errs) > 0 {
t.Fatalf("loading mox config: %v", errs)
}
defer store.Switchboard()()
xlog := mlog.New("ctl")
testctl := func(fn func(clientctl *ctl)) {
t.Helper()
cconn, sconn := net.Pipe()
clientctl := ctl{conn: cconn, log: xlog}
serverctl := ctl{conn: sconn, log: xlog}
clientctl := ctl{conn: cconn, log: pkglog}
serverctl := ctl{conn: sconn, log: pkglog}
go servectlcmd(ctxbg, &serverctl, func() {})
fn(&clientctl)
cconn.Close()
@ -148,8 +147,8 @@ func TestCtl(t *testing.T) {
})
// Export data, import it again
xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, nil)
xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, nil)
xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog})
xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog})
testctl(func(ctl *ctl) {
ctlcmdImport(ctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox"))
})
@ -167,7 +166,7 @@ func TestCtl(t *testing.T) {
ctlcmdFixmsgsize(ctl, "mjl")
})
testctl(func(ctl *ctl) {
acc, err := store.OpenAccount("mjl")
acc, err := store.OpenAccount(ctl.log, "mjl")
tcheck(t, err, "open account")
defer acc.Close()
@ -176,13 +175,13 @@ func TestCtl(t *testing.T) {
deliver := func(m *store.Message) {
t.Helper()
m.Size = int64(len(content))
msgf, err := store.CreateMessageTemp("ctltest")
msgf, err := store.CreateMessageTemp(ctl.log, "ctltest")
tcheck(t, err, "create temp file")
defer os.Remove(msgf.Name())
defer msgf.Close()
_, err = msgf.Write(content)
tcheck(t, err, "write message file")
err = acc.DeliverMailbox(xlog, "Inbox", m, msgf)
err = acc.DeliverMailbox(ctl.log, "Inbox", m, msgf)
tcheck(t, err, "deliver message")
}

22
dane/dane.go
View file

@ -59,6 +59,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -132,8 +134,8 @@ func (e VerifyError) Unwrap() error {
// indicate DNSSEC errors.
// - ErrInsecure
// - VerifyError, potentially wrapping errors from crypto/x509.
func Dial(ctx context.Context, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) {
log := mlog.New("dane").WithContext(ctx)
func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) {
log := mlog.New("dane", elog)
// Split host and port.
host, portstr, err := net.SplitHostPort(address)
@ -272,7 +274,7 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a
}
var verifiedRecord adns.TLSA
config := TLSClientConfig(log, records, baseDom, moreAllowedHosts, &verifiedRecord)
config := TLSClientConfig(log.Logger, records, baseDom, moreAllowedHosts, &verifiedRecord)
tlsConn := tls.Client(conn, &config)
if err := tlsConn.HandshakeContext(ctx); err != nil {
conn.Close()
@ -295,13 +297,14 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a
//
// If verifiedRecord is not nil, it is set to the record that was successfully
// verified, if any.
func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config {
func TLSClientConfig(elog *slog.Logger, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config {
log := mlog.New("dane", elog)
return tls.Config{
ServerName: allowedHost.ASCII, // For SNI.
InsecureSkipVerify: true,
VerifyConnection: func(cs tls.ConnectionState) error {
verified, record, err := Verify(log, records, cs, allowedHost, moreAllowedHosts)
log.Debugx("dane verification", err, mlog.Field("verified", verified), mlog.Field("record", record))
verified, record, err := Verify(log.Logger, records, cs, allowedHost, moreAllowedHosts)
log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record))
if verified {
if verifiedRecord != nil {
*verifiedRecord = record
@ -332,7 +335,8 @@ func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain,
// If an error is encountered while verifying a record, e.g. for x509
// trusted-anchor verification, an error may be returned, typically one or more
// (wrapped) errors of type VerifyError.
func Verify(log *mlog.Log, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) {
func Verify(elog *slog.Logger, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) {
log := mlog.New("dane", elog)
metricVerify.Inc()
if len(records) == 0 {
metricVerifyErrors.Inc()
@ -360,7 +364,7 @@ func Verify(log *mlog.Log, records []adns.TLSA, cs tls.ConnectionState, allowedH
// errors while verifying certificates against a trust-anchor, an error can be
// returned with one or more underlying x509 verification errors. A nil-nil error
// is only returned when verified is false.
func verifySingle(log *mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) {
func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) {
if len(cs.PeerCertificates) == 0 {
return false, fmt.Errorf("no server certificate")
}
@ -513,7 +517,7 @@ func verifySingle(log *mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowed
default:
// Unknown, perhaps defined in the future. Not an error.
log.Debug("unrecognized tlsa usage, skipping", mlog.Field("tlsausage", tlsa.Usage))
log.Debug("unrecognized tlsa usage, skipping", slog.Any("tlsausage", tlsa.Usage))
return false, nil
}
}

8
dane/dane_test.go
View file

@ -17,9 +17,10 @@ import (
"reflect"
"strconv"
"sync/atomic"
"testing"
"time"
"testing"
"golang.org/x/exp/slog"
"github.com/mjl-/adns"
@ -37,7 +38,8 @@ func tcheckf(t *testing.T, err error, format string, args ...any) {
// Test dialing and DANE TLS verification.
func TestDial(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug})
mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug})
log := mlog.New("dane", nil)
// Create fake CA/trusted-anchor certificate.
taTempl := x509.Certificate{
@ -139,7 +141,7 @@ func TestDial(t *testing.T) {
test := func(resolver dns.Resolver, expRecord adns.TLSA, expErr any) {
t.Helper()
conn, record, err := Dial(context.Background(), resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages)
conn, record, err := Dial(context.Background(), log.Logger, resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages)
if err == nil {
conn.Close()
}

8
develop.txt
View file

@ -10,6 +10,14 @@ This file has notes useful for mox developers.
name/path manipulation. Do not remove/rename files that are still open.
- Not all code uses adns, the DNSSEC-aware resolver. Such as code that makes
http requests, like mtasts and autotls/autocert.
- We don't have an internal/ directory, really just to prevent long paths in
the repo, and to keep all Go code matching *.go */*.go (without matching
vendor/). Part of the packages are reusable by other software. Those reusable
packages must not cause mox implementation details (such as bstore) to get out,
which would cause unexpected dependencies. Those packages also only expose the
standard slog package for logging, not our mlog package. Packages not intended
for reuse do use mlog as it is more convenient. Internally, we always use
mlog.Log to do the logging, wrapping an slog.Logger.
# TLS certificates

36
dkim/dkim.go
View file

@ -24,6 +24,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -35,8 +37,6 @@ import (
"github.com/mjl-/mox/smtp"
)
var xlog = mlog.New("dkim")
var (
metricSign = promauto.NewCounterVec(
prometheus.CounterOpts{
@ -123,11 +123,11 @@ type Result struct {
// todo: use some io.Writer to hash the body and the header.
// Sign returns line(s) with DKIM-Signature headers, generated according to the configuration.
func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) {
log := xlog.WithContext(ctx)
func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) {
log := mlog.New("dkim", elog)
start := timeNow()
defer func() {
log.Debugx("dkim sign result", rerr, mlog.Field("localpart", localpart), mlog.Field("domain", domain), mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start)))
log.Debugx("dkim sign result", rerr, slog.Any("localpart", localpart), slog.Any("domain", domain), slog.Bool("smtputf8", smtputf8), slog.Duration("duration", time.Since(start)))
}()
hdrs, bodyOffset, err := parseHeaders(bufio.NewReader(&moxio.AtReader{R: msg}))
@ -270,11 +270,11 @@ func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c co
// record should be present.
//
// authentic indicates if DNS results were DNSSEC-verified.
func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, authentic bool, rerr error) {
log := xlog.WithContext(ctx)
func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, authentic bool, rerr error) {
log := mlog.New("dkim", elog)
start := timeNow()
defer func() {
log.Debugx("dkim lookup result", rerr, mlog.Field("selector", selector), mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start)))
log.Debugx("dkim lookup result", rerr, slog.Any("selector", selector), slog.Any("domain", domain), slog.Any("status", rstatus), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start)))
}()
name := selector.ASCII + "._domainkey." + domain.ASCII + "."
@ -338,8 +338,8 @@ func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Dom
// verification failure is treated as actual failure. With ignoreTestMode
// false, such verification failures are treated as if there is no signature by
// returning StatusNone.
func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) {
log := xlog.WithContext(ctx)
func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) {
log := mlog.New("dkim", elog)
start := timeNow()
defer func() {
duration := float64(time.Since(start)) / float64(time.Second)
@ -353,10 +353,10 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
}
if len(results) == 0 {
log.Debugx("dkim verify result", rerr, mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start)))
log.Debugx("dkim verify result", rerr, slog.Bool("smtputf8", smtputf8), slog.Duration("duration", time.Since(start)))
}
for _, result := range results {
log.Debugx("dkim verify result", result.Err, mlog.Field("smtputf8", smtputf8), mlog.Field("status", result.Status), mlog.Field("sig", result.Sig), mlog.Field("record", result.Record), mlog.Field("duration", time.Since(start)))
log.Debugx("dkim verify result", result.Err, slog.Bool("smtputf8", smtputf8), slog.Any("status", result.Status), slog.Any("sig", result.Sig), slog.Any("record", result.Record), slog.Duration("duration", time.Since(start)))
}
}()
@ -380,7 +380,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
continue
}
h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig)
h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, log, sig)
if err != nil {
results = append(results, Result{StatusPermerror, sig, nil, false, err})
continue
@ -394,7 +394,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
}
br := bufio.NewReader(&moxio.AtReader{R: r, Offset: int64(bodyOffset)})
status, txt, authentic, err := verifySignature(ctx, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode)
status, txt, authentic, err := verifySignature(ctx, log.Logger, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode)
results = append(results, Result{status, sig, txt, authentic, err})
}
return results, nil
@ -402,7 +402,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
// check if signature is acceptable.
// Only looks at the signature parameters, not at the DNS record.
func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) {
func checkSignatureParams(ctx context.Context, log mlog.Log, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) {
// "From" header is required, ../rfc/6376:2122 ../rfc/6376:2546
var from bool
for _, h := range sig.SignedHeaders {
@ -431,7 +431,7 @@ func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, cano
if subdom.Unicode != "" {
subdom.Unicode = "x." + subdom.Unicode
}
if orgDom := publicsuffix.Lookup(ctx, subdom); subdom.ASCII == orgDom.ASCII {
if orgDom := publicsuffix.Lookup(ctx, log.Logger, subdom); subdom.ASCII == orgDom.ASCII {
return 0, false, false, fmt.Errorf("%w: %s", ErrTLD, sig.Domain)
}
@ -480,9 +480,9 @@ func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, cano
}
// lookup the public key in the DNS and verify the signature.
func verifySignature(ctx context.Context, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, bool, error) {
func verifySignature(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, bool, error) {
// ../rfc/6376:2604
status, record, _, authentic, err := Lookup(ctx, resolver, sig.Selector, sig.Domain)
status, record, _, authentic, err := Lookup(ctx, elog, resolver, sig.Selector, sig.Domain)
if err != nil {
// todo: for temporary errors, we could pass on information so caller returns a 4.7.5 ecode, ../rfc/6376:2777
return status, nil, authentic, err

27
dkim/dkim_test.go
View file

@ -17,8 +17,11 @@ import (
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
)
var pkglog = mlog.New("dkim", nil)
func policyOK(sig *Sig) error {
return nil
}
@ -143,7 +146,7 @@ test
},
}
results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false)
results, err := Verify(context.Background(), pkglog.Logger, resolver, false, policyOK, strings.NewReader(message), false)
if err != nil {
t.Fatalf("dkim verify: %v", err)
}
@ -190,7 +193,7 @@ Joe.
},
}
results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false)
results, err := Verify(context.Background(), pkglog.Logger, resolver, false, policyOK, strings.NewReader(message), false)
if err != nil {
t.Fatalf("dkim verify: %v", err)
}
@ -262,7 +265,7 @@ test
}
ctx := context.Background()
headers, err := Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message))
headers, err := Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message))
if err != nil {
t.Fatalf("sign: %v", err)
}
@ -293,7 +296,7 @@ test
nmsg := headers + message
results, err := Verify(ctx, resolver, false, policyOK, strings.NewReader(nmsg), false)
results, err := Verify(ctx, pkglog.Logger, resolver, false, policyOK, strings.NewReader(nmsg), false)
if err != nil {
t.Fatalf("verify: %s", err)
}
@ -304,31 +307,31 @@ test
//log.Infof("nmsg\n%s", nmsg)
// Multiple From headers.
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest"))
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrFrom) {
t.Fatalf("sign, got err %v, expected ErrFrom", err)
}
// No From header.
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest"))
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrFrom) {
t.Fatalf("sign, got err %v, expected ErrFrom", err)
}
// Malformed headers.
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest"))
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
}
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest"))
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
}
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest"))
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
}
_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:<mjl@mox.example>"))
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:<mjl@mox.example>"))
if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
}
@ -408,7 +411,7 @@ test
msg = strings.ReplaceAll(msg, "\n", "\r\n")
headers, err := Sign(context.Background(), "mjl", signDomain, dkimConf, false, strings.NewReader(msg))
headers, err := Sign(context.Background(), pkglog.Logger, "mjl", signDomain, dkimConf, false, strings.NewReader(msg))
if err != nil {
t.Fatalf("sign: %v", err)
}
@ -425,7 +428,7 @@ test
sign()
}
results, err := Verify(context.Background(), resolver, true, policy, strings.NewReader(msg), false)
results, err := Verify(context.Background(), pkglog.Logger, resolver, true, policy, strings.NewReader(msg), false)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got verify error %v, expected %v", err, expErr)
}

28
dmarc/dmarc.go
View file

@ -25,9 +25,9 @@ import (
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/spf"
)
var xlog = mlog.New("dmarc")
"golang.org/x/exp/slog"
)
var (
metricDMARCVerify = promauto.NewHistogramVec(
@ -99,11 +99,11 @@ type Result struct {
// domain is the domain with the DMARC record.
//
// rauthentic indicates if the DNS results were DNSSEC-verified.
func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rauthentic bool, rerr error) {
log := xlog.WithContext(ctx)
func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rauthentic bool, rerr error) {
log := mlog.New("dmarc", elog)
start := time.Now()
defer func() {
log.Debugx("dmarc lookup result", rerr, mlog.Field("fromdomain", from), mlog.Field("status", status), mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("duration", time.Since(start)))
log.Debugx("dmarc lookup result", rerr, slog.Any("fromdomain", from), slog.Any("status", status), slog.Any("domain", domain), slog.Any("record", record), slog.Duration("duration", time.Since(start)))
}()
// ../rfc/7489:859 ../rfc/7489:1370
@ -114,7 +114,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status
}
if record == nil {
// ../rfc/7489:761 ../rfc/7489:1377
domain = publicsuffix.Lookup(ctx, from)
domain = publicsuffix.Lookup(ctx, log.Logger, from)
if domain == from {
return StatusNone, domain, nil, txt, authentic, err
}
@ -202,11 +202,11 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain
// example in RFC 7489.
//
// authentic indicates if the DNS results were DNSSEC-verified.
func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) {
log := xlog.WithContext(ctx)
func LookupExternalReportsAccepted(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) {
log := mlog.New("dmarc", elog)
start := time.Now()
defer func() {
log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start)))
log.Debugx("dmarc externalreports result", rerr, slog.Bool("accepts", accepts), slog.Any("dmarcdomain", dmarcDomain), slog.Any("extdestdomain", extDestDomain), slog.Any("records", records), slog.Duration("duration", time.Since(start)))
}()
status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
@ -226,8 +226,8 @@ func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, d
// against the message (for inclusion in Authentication-Result headers).
//
// useResult indicates if the result should be applied in a policy decision.
func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) {
log := xlog.WithContext(ctx)
func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) {
log := mlog.New("dmarc", elog)
start := time.Now()
defer func() {
use := "no"
@ -239,10 +239,10 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
reject = "yes"
}
metricDMARCVerify.WithLabelValues(string(result.Status), reject, use).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("dmarc verify result", result.Err, mlog.Field("fromdomain", from), mlog.Field("dkimresults", dkimResults), mlog.Field("spfresult", spfResult), mlog.Field("status", result.Status), mlog.Field("reject", result.Reject), mlog.Field("use", useResult), mlog.Field("duration", time.Since(start)))
log.Debugx("dmarc verify result", result.Err, slog.Any("fromdomain", from), slog.Any("dkimresults", dkimResults), slog.Any("spfresult", spfResult), slog.Any("status", result.Status), slog.Bool("reject", result.Reject), slog.Bool("use", useResult), slog.Duration("duration", time.Since(start)))
}()
status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from)
status, recordDomain, record, _, authentic, err := Lookup(ctx, log.Logger, resolver, from)
if record == nil {
return false, Result{false, status, false, false, recordDomain, record, authentic, err}
}
@ -277,7 +277,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
if r, ok := pubsuffixes[name]; ok {
return r
}
r := publicsuffix.Lookup(ctx, name)
r := publicsuffix.Lookup(ctx, log.Logger, name)
pubsuffixes[name] = r
return r
}

9
dmarc/dmarc_test.go
View file

@ -8,9 +8,12 @@ import (
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/spf"
)
var pkglog = mlog.New("dmarc", nil)
func TestLookup(t *testing.T) {
resolver := dns.MockResolver{
TXT: map[string][]string{
@ -29,7 +32,7 @@ func TestLookup(t *testing.T) {
test := func(d string, expStatus Status, expDomain string, expRecord *Record, expErr error) {
t.Helper()
status, dom, record, _, _, err := Lookup(context.Background(), resolver, dns.Domain{ASCII: d})
status, dom, record, _, _, err := Lookup(context.Background(), pkglog.Logger, resolver, dns.Domain{ASCII: d})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %#v, expected %#v", err, expErr)
}
@ -68,7 +71,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) {
test := func(dom, extdom string, expStatus Status, expAccepts bool, expErr error) {
t.Helper()
accepts, status, _, _, _, err := LookupExternalReportsAccepted(context.Background(), resolver, dns.Domain{ASCII: dom}, dns.Domain{ASCII: extdom})
accepts, status, _, _, _, err := LookupExternalReportsAccepted(context.Background(), pkglog.Logger, resolver, dns.Domain{ASCII: dom}, dns.Domain{ASCII: extdom})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %#v, expected %#v", err, expErr)
}
@ -124,7 +127,7 @@ func TestVerify(t *testing.T) {
if err != nil {
t.Fatalf("parsing domain: %v", err)
}
useResult, result := Verify(context.Background(), resolver, from, dkimResults, spfResult, spfIdentity, true)
useResult, result := Verify(context.Background(), pkglog.Logger, resolver, from, dkimResults, spfResult, spfIdentity, true)
if useResult != expUseResult || !equalResult(result, expResult) {
t.Fatalf("verify: got useResult %v, result %#v, expected %v %#v", useResult, result, expUseResult, expResult)
}

69
dmarcdb/eval.go
View file

@ -23,6 +23,7 @@ import (
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -300,13 +301,13 @@ var jitteredTimeUntil = func(t time.Time) time.Duration {
// sends DMARC reports to domains that requested them.
func Start(resolver dns.Resolver) {
go func() {
log := mlog.New("dmarcdb")
log := mlog.New("dmarcdb", nil)
defer func() {
// In case of panic don't take the whole program down.
x := recover()
if x != nil {
log.Error("recover from panic", mlog.Field("panic", x))
log.Error("recover from panic", slog.Any("panic", x))
debug.PrintStack()
metrics.PanicInc(metrics.Dmarcdb)
}
@ -358,7 +359,7 @@ func Start(resolver dns.Resolver) {
log.Check(err, "removing stale dmarc evaluations from database")
clog := log.WithCid(mox.Cid())
clog.Info("sending dmarc aggregate reports", mlog.Field("end", nextEnd.UTC()), mlog.Field("intervals", intervals))
clog.Info("sending dmarc aggregate reports", slog.Time("end", nextEnd.UTC()), slog.Any("intervals", intervals))
if err := sendReports(ctx, clog, resolver, db, nextEnd, intervals); err != nil {
clog.Errorx("sending dmarc aggregate reports", err)
metricReportError.Inc()
@ -393,7 +394,7 @@ var sleepBetween = func(ctx context.Context, between time.Duration) (ok bool) {
// sendReports gathers all policy domains that have evaluations that should
// receive a DMARC report and sends a report to each.
func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error {
func sendReports(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error {
ivals := make([]any, len(intervals))
for i, v := range intervals {
ivals[i] = v
@ -452,14 +453,14 @@ func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *
// In case of panic don't take the whole program down.
x := recover()
if x != nil {
log.Error("unhandled panic in dmarcdb sendReports", mlog.Field("panic", x))
log.Error("unhandled panic in dmarcdb sendReports", slog.Any("panic", x))
debug.PrintStack()
metrics.PanicInc(metrics.Dmarcdb)
}
}()
defer wg.Done()
rlog := log.WithCid(mox.Cid()).Fields(mlog.Field("domain", domain))
rlog := log.WithCid(mox.Cid()).With(slog.Any("domain", domain))
rlog.Info("sending dmarc report")
if _, err := sendReportDomain(ctx, rlog, resolver, db, endTime, domain); err != nil {
rlog.Errorx("sending dmarc aggregate report to domain", err)
@ -478,8 +479,8 @@ type recipient struct {
maxSize uint64
}
func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) {
log = log.Fields(mlog.Field("uri", uri.Address))
func parseRecipient(log mlog.Log, uri dmarc.URI) (r recipient, ok bool) {
log = log.With(slog.Any("uri", uri.Address))
u, err := url.Parse(uri.Address)
if err != nil {
@ -510,14 +511,14 @@ func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) {
r.maxSize *= 1024 * 1024 * 1024 * 1024
case "":
default:
log.Debug("unrecognized max size unit in dmarc record rua value", mlog.Field("unit", uri.Unit))
log.Debug("unrecognized max size unit in dmarc record rua value", slog.String("unit", uri.Unit))
return r, false
}
return r, true
}
func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTime time.Time, domain string) {
func removeEvaluations(ctx context.Context, log mlog.Log, db *bstore.DB, endTime time.Time, domain string) {
q := bstore.QueryDB[Evaluation](ctx, db)
q.FilterLess("Evaluated", endTime)
q.FilterNonzero(Evaluation{PolicyDomain: domain})
@ -528,7 +529,7 @@ func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTim
// replaceable for testing.
var queueAdd = queue.Add
func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) {
func sendReportDomain(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) {
dom, err := dns.ParseDomain(domain)
if err != nil {
return false, fmt.Errorf("parsing domain for sending reports: %v", err)
@ -567,7 +568,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// evaluations regardless. We always use the latest DMARC record when sending, but
// we'll lump all policies of the last interval into one report.
// ../rfc/7489:1714
status, _, record, _, _, err := dmarc.Lookup(ctx, resolver, dom)
status, _, record, _, _, err := dmarc.Lookup(ctx, log.Logger, resolver, dom)
if err != nil {
// todo future: we could perhaps still send this report, assuming the values we know. in case of temporary error, we could also schedule again regardless of next interval hour (we would now only retry a 24h-interval report after 24h passed).
// Remove records unless it was a temporary error. We'll try again next round.
@ -588,8 +589,8 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// Check if domain of rua recipient has the same organizational domain as for the
// evaluations. If not, we need to verify we are allowed to send.
ruaOrgDom := publicsuffix.Lookup(ctx, r.address.Domain)
evalOrgDom := publicsuffix.Lookup(ctx, dom)
ruaOrgDom := publicsuffix.Lookup(ctx, log.Logger, r.address.Domain)
evalOrgDom := publicsuffix.Lookup(ctx, log.Logger, dom)
if ruaOrgDom == evalOrgDom {
recipients = append(recipients, r)
@ -599,12 +600,12 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// Verify and follow addresses in other organizational domain through
// <policydomain>._report._dmarc.<host> lookup.
// ../rfc/7489:1556
accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, resolver, evalOrgDom, r.address.Domain)
accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, log.Logger, resolver, evalOrgDom, r.address.Domain)
log.Debugx("checking if rua address with different organization domain has opted into receiving dmarc reports", err,
mlog.Field("policydomain", evalOrgDom),
mlog.Field("destinationdomain", r.address.Domain),
mlog.Field("accepts", accepts),
mlog.Field("status", status))
slog.Any("policydomain", evalOrgDom),
slog.Any("destinationdomain", r.address.Domain),
slog.Bool("accepts", accepts),
slog.Any("status", status))
if status == dmarc.StatusTemperror {
// With a temporary error, we'll try to get the report the delivered anyway,
// perhaps there are multiple recipients.
@ -626,7 +627,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
// alternative addresses and no new address specified).
// ../rfc/7489:1600
foundReplacement := false
rlog := log.Fields(mlog.Field("followedaddress", uri.Address))
rlog := log.With(slog.Any("followedaddress", uri.Address))
for _, record := range records {
for _, exturi := range record.AggregateReportAddresses {
extr, ok := parseRecipient(rlog, exturi)
@ -634,10 +635,10 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
continue
}
if extr.address.Domain != r.address.Domain {
rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", mlog.Field("externaladdress", extr.address))
rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", slog.Any("externaladdress", extr.address))
errors = append(errors, fmt.Sprintf("rua %s is external domain with a replacement address %s with different host", r.address, extr.address))
} else {
rlog.Debug("using replacement rua address from external _report dmarc record", mlog.Field("externaladdress", extr.address))
rlog.Debug("using replacement rua address from external _report dmarc record", slog.Any("externaladdress", extr.address))
foundReplacement = true
recipients = append(recipients, extr)
}
@ -744,7 +745,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
report.Records = append(report.Records, rc.ReportRecord)
}
reportFile, err := store.CreateMessageTemp("dmarcreportout")
reportFile, err := store.CreateMessageTemp(log, "dmarcreportout")
if err != nil {
return false, fmt.Errorf("creating temporary file for outgoing dmarc aggregate report: %v", err)
}
@ -767,7 +768,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
return true, fmt.Errorf("writing dmarc aggregate report as xml with gzip: %v", err)
}
msgf, err := store.CreateMessageTemp("dmarcreportmsgout")
msgf, err := store.CreateMessageTemp(log, "dmarcreportmsgout")
if err != nil {
return false, fmt.Errorf("creating temporary message file with outgoing dmarc aggregate report: %v", err)
}
@ -828,7 +829,7 @@ Period: %s - %s UTC
return false, fmt.Errorf("querying suppress list: %v", err)
}
if exists {
log.Info("suppressing outgoing dmarc aggregate report", mlog.Field("reportingaddress", rcpt.address))
log.Info("suppressing outgoing dmarc aggregate report", slog.Any("reportingaddress", rcpt.address))
continue
}
@ -853,7 +854,7 @@ Period: %s - %s UTC
log.Errorx("queueing message with dmarc aggregate report", err)
metricReportError.Inc()
} else {
log.Debug("dmarc aggregate report queued", mlog.Field("recipient", rcpt.address))
log.Debug("dmarc aggregate report queued", slog.Any("recipient", rcpt.address))
queued = true
metricReport.Inc()
}
@ -873,7 +874,7 @@ Period: %s - %s UTC
return true, nil
}
func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) {
func composeAggregateReport(ctx context.Context, log mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) {
xc := message.NewComposer(mf, 100*1024*1024)
defer func() {
x := recover()
@ -946,10 +947,10 @@ func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fro
// Though this functionality is quite underspecified, we'll do our best to send our
// an error report in case our report is too large for all recipients.
// ../rfc/7489:1918
func sendErrorReport(ctx context.Context, log *mlog.Log, db *bstore.DB, fromAddr smtp.Address, recipients []message.NameAddress, reportDomain dns.Domain, reportID string, reportMsgSize int64) error {
func sendErrorReport(ctx context.Context, log mlog.Log, db *bstore.DB, fromAddr smtp.Address, recipients []message.NameAddress, reportDomain dns.Domain, reportID string, reportMsgSize int64) error {
log.Debug("no reporting addresses willing to accept report given size, queuing short error message")
msgf, err := store.CreateMessageTemp("dmarcreportmsg-out")
msgf, err := store.CreateMessageTemp(log, "dmarcreportmsg-out")
if err != nil {
return fmt.Errorf("creating temporary message file for outgoing dmarc error report: %v", err)
}
@ -992,7 +993,7 @@ Submitting-URI: %s
return fmt.Errorf("querying suppress list: %v", err)
}
if exists {
log.Info("suppressing outgoing dmarc error report", mlog.Field("reportingaddress", rcpt.Address))
log.Info("suppressing outgoing dmarc error report", slog.Any("reportingaddress", rcpt.Address))
continue
}
@ -1006,14 +1007,14 @@ Submitting-URI: %s
log.Errorx("queueing message with dmarc error report", err)
metricReportError.Inc()
} else {
log.Debug("dmarc error report queued", mlog.Field("recipient", rcpt))
log.Debug("dmarc error report queued", slog.Any("recipient", rcpt))
metricReport.Inc()
}
}
return nil
}
func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) {
func composeErrorReport(ctx context.Context, log mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) {
xc := message.NewComposer(mf, 100*1024*1024)
defer func() {
x := recover()
@ -1058,7 +1059,7 @@ func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAdd
return msgPrefix, xc.Has8bit, xc.SMTPUTF8, messageID, nil
}
func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string {
func dkimSign(ctx context.Context, log mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string {
// Add DKIM-Signature headers if we have a key for (a higher) domain than the from
// address, which is a host name. A signature will only be useful with higher-level
// domains if they have a relaxed dkim check (which is the default). If the dkim
@ -1068,7 +1069,7 @@ func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf
for fd != zerodom {
confDom, ok := mox.Conf.Domain(fd)
if len(confDom.DKIM.Sign) > 0 {
dkimHeaders, err := dkim.Sign(ctx, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf)
dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf)
if err != nil {
log.Errorx("dkim-signing dmarc report, continuing without signature", err)
metricReportError.Inc()

8
dmarcdb/eval_test.go
View file

@ -13,6 +13,8 @@ import (
"testing"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
@ -156,7 +158,7 @@ func TestEvaluations(t *testing.T) {
}
func TestSendReports(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug})
mlog.SetConfig(map[string]slog.Level{"": slog.LevelDebug})
os.RemoveAll("../testdata/dmarcdb/data")
mox.Context = ctxbg
@ -294,7 +296,7 @@ func TestSendReports(t *testing.T) {
aggrAddrs := map[string]struct{}{}
errorAddrs := map[string]struct{}{}
queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error {
queueAdd = func(ctx context.Context, log mlog.Log, qm *queue.Msg, msgFile *os.File) error {
// Read message file. Also write copy to disk for inspection.
buf, err := io.ReadAll(&moxio.AtReader{R: msgFile})
tcheckf(t, err, "read report message")
@ -309,7 +311,7 @@ func TestSendReports(t *testing.T) {
} else {
aggrAddrs[addr] = struct{}{}
feedback, err = dmarcrpt.ParseMessageReport(log, msgFile)
feedback, err = dmarcrpt.ParseMessageReport(log.Logger, msgFile)
tcheckf(t, err, "parsing generated report message")
}

11
dmarcrpt/parse.go
View file

@ -12,6 +12,8 @@ import (
"net/http"
"strings"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
@ -34,9 +36,10 @@ func ParseReport(r io.Reader) (*Feedback, error) {
// ParseMessageReport parses an aggregate feedback report from a mail message. The
// maximum message size is 15MB, the maximum report size after decompression is
// 20MB.
func ParseMessageReport(log *mlog.Log, r io.ReaderAt) (*Feedback, error) {
func ParseMessageReport(elog *slog.Logger, r io.ReaderAt) (*Feedback, error) {
log := mlog.New("dmarcrpt", elog)
// ../rfc/7489:1801
p, err := message.Parse(log, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024})
p, err := message.Parse(log.Logger, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024})
if err != nil {
return nil, fmt.Errorf("parsing mail message: %s", err)
}
@ -44,7 +47,7 @@ func ParseMessageReport(log *mlog.Log, r io.ReaderAt) (*Feedback, error) {
return parseMessageReport(log, p)
}
func parseMessageReport(log *mlog.Log, p message.Part) (*Feedback, error) {
func parseMessageReport(log mlog.Log, p message.Part) (*Feedback, error) {
// Pretty much any mime structure is allowed. ../rfc/7489:1861
// In practice, some parties will send the report as the only (non-multipart)
// content of the message.
@ -54,7 +57,7 @@ func parseMessageReport(log *mlog.Log, p message.Part) (*Feedback, error) {
}
for {
sp, err := p.ParseNextPart(log)
sp, err := p.ParseNextPart(log.Logger)
if err == io.EOF {
return nil, ErrNoReport
}

8
dmarcrpt/parse_test.go
View file

@ -11,7 +11,7 @@ import (
"github.com/mjl-/mox/mlog"
)
var xlog = mlog.New("dmarcrpt")
var pkglog = mlog.New("dmarcrpt", nil)
const reportExample = `<?xml version="1.0" encoding="UTF-8" ?>
<feedback>
@ -137,7 +137,7 @@ func TestParseMessageReport(t *testing.T) {
if err != nil {
t.Fatalf("open %q: %s", p, err)
}
_, err = ParseMessageReport(xlog, f)
_, err = ParseMessageReport(pkglog.Logger, f)
if err != nil {
t.Fatalf("ParseMessageReport: %q: %s", p, err)
}
@ -145,7 +145,7 @@ func TestParseMessageReport(t *testing.T) {
}
// No report in a non-multipart message.
_, err = ParseMessageReport(xlog, strings.NewReader("From: <mjl@mox.example>\r\n\r\nNo report.\r\n"))
_, err = ParseMessageReport(pkglog.Logger, strings.NewReader("From: <mjl@mox.example>\r\n\r\nNo report.\r\n"))
if err != ErrNoReport {
t.Fatalf("message without report, got err %#v, expected ErrNoreport", err)
}
@ -171,7 +171,7 @@ MIME-Version: 1.0
--===============5735553800636657282==--
`, "\n", "\r\n")
_, err = ParseMessageReport(xlog, strings.NewReader(multipartNoreport))
_, err = ParseMessageReport(pkglog.Logger, strings.NewReader(multipartNoreport))
if err != ErrNoReport {
t.Fatalf("message without report, got err %#v, expected ErrNoreport", err)
}

168
dns/resolver.go
View file

@ -10,6 +10,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -22,8 +24,6 @@ import (
// todo future: change to interface that is closer to DNS. 1. expose nxdomain vs success with zero entries: nxdomain means the name does not exist for any dns resource record type, success with zero records means the name exists for other types than the requested type; 2. add ability to not follow cname records when resolving. the net resolver automatically follows cnames for LookupHost, LookupIP, LookupIPAddr. when resolving names found in mx records, we explicitly must not follow cnames. that seems impossible at the moment. 3. when looking up a cname, actually lookup the record? "net" LookupCNAME will return the requested name with no error if there is no CNAME record. because it returns the canonical name.
// todo future: add option to not use anything in the cache, for the admin pages where you check the latest DNS settings, ignoring old cached info.
var xlog = mlog.New("dns")
func init() {
net.DefaultResolver.StrictErrors = true
}
@ -74,6 +74,15 @@ func WithPackage(resolver Resolver, name string) Resolver {
type StrictResolver struct {
Pkg string // Name of subsystem that is making DNS requests, for metrics.
Resolver *adns.Resolver // Where the actual lookups are done. If nil, adns.DefaultResolver is used for lookups.
Log *slog.Logger
}
func (r StrictResolver) log() mlog.Log {
pkg := r.Pkg
if pkg == "" {
pkg = "dns"
}
return mlog.New(pkg, r.Log)
}
var _ Resolver = StrictResolver{}
@ -133,13 +142,12 @@ func (r StrictResolver) LookupPort(ctx context.Context, network, service string)
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "port", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "port"),
mlog.Field("network", network),
mlog.Field("service", service),
mlog.Field("resp", resp),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "port"),
slog.String("network", network),
slog.String("service", service),
slog.Int("resp", resp),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -152,13 +160,12 @@ func (r StrictResolver) LookupAddr(ctx context.Context, addr string) (resp []str
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "addr", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "addr"),
mlog.Field("addr", addr),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "addr"),
slog.String("addr", addr),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -179,13 +186,12 @@ func (r StrictResolver) LookupCNAME(ctx context.Context, host string) (resp stri
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "cname", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "cname"),
mlog.Field("host", host),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "cname"),
slog.String("host", host),
slog.String("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -209,13 +215,12 @@ func (r StrictResolver) LookupHost(ctx context.Context, host string) (resp []str
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "host", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "host"),
mlog.Field("host", host),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "host"),
slog.String("host", host),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -231,14 +236,13 @@ func (r StrictResolver) LookupIP(ctx context.Context, network, host string) (res
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "ip", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "ip"),
mlog.Field("network", network),
mlog.Field("host", host),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "ip"),
slog.String("network", network),
slog.String("host", host),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -254,13 +258,12 @@ func (r StrictResolver) LookupIPAddr(ctx context.Context, host string) (resp []n
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "ipaddr", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "ipaddr"),
mlog.Field("host", host),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "ipaddr"),
slog.String("host", host),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -276,13 +279,12 @@ func (r StrictResolver) LookupMX(ctx context.Context, name string) (resp []*net.
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "mx", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "mx"),
mlog.Field("name", name),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "mx"),
slog.String("name", name),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -298,13 +300,12 @@ func (r StrictResolver) LookupNS(ctx context.Context, name string) (resp []*net.
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "ns", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "ns"),
mlog.Field("name", name),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "ns"),
slog.String("name", name),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -320,16 +321,15 @@ func (r StrictResolver) LookupSRV(ctx context.Context, service, proto, name stri
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "srv", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "srv"),
mlog.Field("service", service),
mlog.Field("proto", proto),
mlog.Field("name", name),
mlog.Field("resp0", resp0),
mlog.Field("resp1", resp1),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "srv"),
slog.String("service", service),
slog.String("proto", proto),
slog.String("name", name),
slog.String("resp0", resp0),
slog.Any("resp1", resp1),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -345,13 +345,12 @@ func (r StrictResolver) LookupTXT(ctx context.Context, name string) (resp []stri
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "txt", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "txt"),
mlog.Field("name", name),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "txt"),
slog.String("name", name),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)
@ -367,15 +366,14 @@ func (r StrictResolver) LookupTLSA(ctx context.Context, port int, protocol, host
start := time.Now()
defer func() {
metricLookupObserve(r.Pkg, "tlsa", err, start)
xlog.WithContext(ctx).Debugx("dns lookup result", err,
mlog.Field("pkg", r.Pkg),
mlog.Field("type", "tlsa"),
mlog.Field("port", port),
mlog.Field("protocol", protocol),
mlog.Field("host", host),
mlog.Field("resp", resp),
mlog.Field("authentic", result.Authentic),
mlog.Field("duration", time.Since(start)),
r.log().WithContext(ctx).Debugx("dns lookup result", err,
slog.String("type", "tlsa"),
slog.Int("port", port),
slog.String("protocol", protocol),
slog.String("host", host),
slog.Any("resp", resp),
slog.Bool("authentic", result.Authentic),
slog.Duration("duration", time.Since(start)),
)
}()
defer resolveErrorHint(&err)

22
dnsbl/dnsbl.go
View file

@ -10,6 +10,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -17,8 +19,6 @@ import (
"github.com/mjl-/mox/mlog"
)
var xlog = mlog.New("dnsbl")
var (
metricLookup = promauto.NewHistogramVec(
prometheus.HistogramOpts{
@ -45,12 +45,12 @@ var (
)
// Lookup checks if "ip" occurs in the DNS block list "zone" (e.g. dnsbl.example.org).
func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) {
log := xlog.WithContext(ctx)
func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) {
log := mlog.New("dnsbl", elog)
start := time.Now()
defer func() {
metricLookup.WithLabelValues(zone.Name(), string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("dnsbl lookup result", rerr, mlog.Field("zone", zone), mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start)))
log.Debugx("dnsbl lookup result", rerr, slog.Any("zone", zone), slog.Any("ip", ip), slog.Any("status", rstatus), slog.String("explanation", rexplanation), slog.Duration("duration", time.Since(start)))
}()
b := &strings.Builder{}
@ -93,7 +93,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.
if dns.IsNotFound(err) {
return StatusFail, "", nil
} else if err != nil {
log.Debugx("looking up txt record from dnsbl", err, mlog.Field("addr", addr))
log.Debugx("looking up txt record from dnsbl", err, slog.String("addr", addr))
return StatusFail, "", nil
}
return StatusFail, strings.Join(txts, "; "), nil
@ -104,16 +104,16 @@ func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.
// Users of a DNSBL should periodically check if the DNSBL is still operating
// properly.
// For temporary errors, ErrDNS is returned.
func CheckHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rerr error) {
log := xlog.WithContext(ctx)
func CheckHealth(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone dns.Domain) (rerr error) {
log := mlog.New("dnsbl", elog)
start := time.Now()
defer func() {
log.Debugx("dnsbl healthcheck result", rerr, mlog.Field("zone", zone), mlog.Field("duration", time.Since(start)))
log.Debugx("dnsbl healthcheck result", rerr, slog.Any("zone", zone), slog.Duration("duration", time.Since(start)))
}()
// ../rfc/5782:355
status1, _, err1 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 1))
status2, _, err2 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 2))
status1, _, err1 := Lookup(ctx, log.Logger, resolver, zone, net.IPv4(127, 0, 0, 1))
status2, _, err2 := Lookup(ctx, log.Logger, resolver, zone, net.IPv4(127, 0, 0, 2))
if status1 == StatusPass && status2 == StatusFail {
return nil
} else if status1 == StatusFail {

14
dnsbl/dnsbl_test.go
View file

@ -6,10 +6,12 @@ import (
"testing"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
)
func TestDNSBL(t *testing.T) {
ctx := context.Background()
log := mlog.New("dnsbl", nil)
resolver := dns.MockResolver{
A: map[string][]string{
@ -23,7 +25,7 @@ func TestDNSBL(t *testing.T) {
},
}
if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil {
if status, expl, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil {
t.Fatalf("lookup: %v", err)
} else if status != StatusFail {
t.Fatalf("lookup, got status %v, expected fail", status)
@ -31,7 +33,7 @@ func TestDNSBL(t *testing.T) {
t.Fatalf("lookup, got explanation %q", expl)
}
if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil {
if status, expl, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil {
t.Fatalf("lookup: %v", err)
} else if status != StatusFail {
t.Fatalf("lookup, got status %v, expected fail", status)
@ -39,17 +41,17 @@ func TestDNSBL(t *testing.T) {
t.Fatalf("lookup, got explanation %q", expl)
}
if status, _, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil {
if status, _, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil {
t.Fatalf("lookup: %v", err)
} else if status != StatusPass {
t.Fatalf("lookup, got status %v, expected pass", status)
}
// ../rfc/5782:357
if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.com"}); err != nil {
if err := CheckHealth(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}); err != nil {
t.Fatalf("dnsbl not healthy: %v", err)
}
if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.org"}); err == nil {
if err := CheckHealth(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.org"}); err == nil {
t.Fatalf("bad dnsbl is healthy")
}
@ -58,7 +60,7 @@ func TestDNSBL(t *testing.T) {
"1.0.0.127.example.com.": {"127.0.0.2"}, // Should not be present in healthy dnsbl.
},
}
if err := CheckHealth(ctx, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil {
if err := CheckHealth(ctx, log.Logger, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil {
t.Fatalf("bad dnsbl is healthy")
}
}

8
dsn/dsn.go
View file

@ -16,6 +16,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message"
@ -135,7 +137,7 @@ type Recipient struct {
// DSN.
//
// DKIM signatures are added if DKIM signing is configured for the "from" domain.
func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) {
func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) {
// ../rfc/3462:119
// ../rfc/3464:377
// We'll make a multipart/report with 2 or 3 parts:
@ -381,9 +383,9 @@ func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) {
continue
}
dkimHeaders, err := dkim.Sign(context.Background(), m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data))
dkimHeaders, err := dkim.Sign(context.Background(), log.Logger, m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data))
if err != nil {
log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, mlog.Field("domain", fd))
log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, slog.Any("domain", fd))
} else {
data = append([]byte(dkimHeaders), data...)
}

8
dsn/dsn_test.go
View file

@ -20,7 +20,7 @@ import (
"github.com/mjl-/mox/smtp"
)
var xlog = mlog.New("dsn")
var pkglog = mlog.New("dsn", nil)
func xparseDomain(s string) dns.Domain {
d, err := dns.ParseDomain(s)
@ -36,7 +36,7 @@ func xparseIPDomain(s string) dns.IPDomain {
func tparseMessage(t *testing.T, data []byte, nparts int) (*Message, *message.Part) {
t.Helper()
m, p, err := Parse(xlog, bytes.NewReader(data))
m, p, err := Parse(pkglog.Logger, bytes.NewReader(data))
if err != nil {
t.Fatalf("parsing dsn: %v", err)
}
@ -75,7 +75,7 @@ func tcompareReader(t *testing.T, r io.Reader, exp []byte) {
}
func TestDSN(t *testing.T) {
log := mlog.New("dsn")
log := mlog.New("dsn", nil)
now := time.Now()
@ -143,7 +143,7 @@ func TestDSN(t *testing.T) {
"testsel._domainkey.mox.example.": {"v=DKIM1;h=sha256;t=s;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3ZId3ys70VFspp/VMFaxMOrNjHNPg04NOE1iShih16b3Ex7hHBOgC1UvTGSmrMlbCB1OxTXkvf6jW6S4oYRnZYVNygH6zKUwYYhaSaGIg1xA/fDn+IgcTRyLoXizMUgUgpTGyxhNrwIIWv+i7jjbs3TKpP3NU4owQ/rxowmSNqg+fHIF1likSvXvljYS" + "jaFXXnWfYibW7TdDCFFpN4sB5o13+as0u4vLw6MvOi59B1tLype1LcHpi1b9PfxNtznTTdet3kL0paxIcWtKHT0LDPUos8YYmiPa5nGbUqlC7d+4YT2jQPvwGxCws1oo2Tw6nj1UaihneYGAyvEky49FBwIDAQAB"},
},
}
results, err := dkim.Verify(context.Background(), resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false)
results, err := dkim.Verify(context.Background(), log.Logger, resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false)
if err != nil {
t.Fatalf("dkim verify: %v", err)
}

10
dsn/parse.go
View file

@ -9,6 +9,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
@ -23,17 +25,19 @@ import (
// The first return value is the machine-parsed DSN message. The second value is
// the entire MIME multipart message. Use its Parts field to access the
// human-readable text and optional original message/headers.
func Parse(log *mlog.Log, r io.ReaderAt) (*Message, *message.Part, error) {
func Parse(elog *slog.Logger, r io.ReaderAt) (*Message, *message.Part, error) {
log := mlog.New("dsn", elog)
// DSNs can mix and match subtypes with and without utf-8. ../rfc/6533:441
part, err := message.Parse(log, false, r)
part, err := message.Parse(log.Logger, false, r)
if err != nil {
return nil, nil, fmt.Errorf("parsing message: %v", err)
}
if part.MediaType != "MULTIPART" || part.MediaSubType != "REPORT" {
return nil, nil, fmt.Errorf(`message has content-type %q, must have "message/report"`, strings.ToLower(part.MediaType+"/"+part.MediaSubType))
}
err = part.Walk(log, nil)
err = part.Walk(log.Logger, nil)
if err != nil {
return nil, nil, fmt.Errorf("parsing message parts: %v", err)
}

3
export.go
View file

@ -8,7 +8,6 @@ import (
"github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/store"
)
@ -66,7 +65,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
}()
a := store.DirArchiver{Dir: dst}
err = store.ExportMessages(context.Background(), mlog.New("export"), db, accountDir, a, !mbox, mailbox)
err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox)
xcheckf(err, "exporting messages")
err = a.Close()
xcheckf(err, "closing archiver")

23
gentestdata.go
View file

@ -54,7 +54,6 @@ func cmdGentestdata(c *cmd) {
return f
}
log := mlog.New("gentestdata")
ctxbg := context.Background()
mox.Conf.Log[""] = mlog.LevelInfo
mlog.SetConfig(mox.Conf.Log)
@ -217,7 +216,7 @@ Accounts:
xcheckf(err, "tlsrptdb init")
tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
xcheckf(err, "parsing tls report")
err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr)
err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr)
xcheckf(err, "adding tls report")
// Populate queue, with a message.
@ -234,22 +233,22 @@ Accounts:
_, err = fmt.Fprint(mf, qmsg)
xcheckf(err, "writing message")
qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
err = queue.Add(ctxbg, log, &qm, mf)
err = queue.Add(ctxbg, c.log, &qm, mf)
xcheckf(err, "enqueue message")
// Create three accounts.
// First account without messages.
accTest0, err := store.OpenAccount("test0")
accTest0, err := store.OpenAccount(c.log, "test0")
xcheckf(err, "open account test0")
err = accTest0.ThreadingWait(log)
err = accTest0.ThreadingWait(c.log)
xcheckf(err, "wait for threading to finish")
err = accTest0.Close()
xcheckf(err, "close account")
// Second account with one message.
accTest1, err := store.OpenAccount("test1")
accTest1, err := store.OpenAccount(c.log, "test1")
xcheckf(err, "open account test1")
err = accTest1.ThreadingWait(log)
err = accTest1.ThreadingWait(c.log)
xcheckf(err, "wait for threading to finish")
err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
@ -285,7 +284,7 @@ Accounts:
xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf, msg)
xcheckf(err, "writing deliver message to file")
err = accTest1.DeliverMessage(log, tx, &m, mf, false, true, false)
err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false)
mfname := mf.Name()
xcheckf(err, "add message to account test1")
@ -307,9 +306,9 @@ Accounts:
xcheckf(err, "close account")
// Third account with two messages and junkfilter.
accTest2, err := store.OpenAccount("test2")
accTest2, err := store.OpenAccount(c.log, "test2")
xcheckf(err, "open account test2")
err = accTest2.ThreadingWait(log)
err = accTest2.ThreadingWait(c.log)
xcheckf(err, "wait for threading to finish")
err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
@ -345,7 +344,7 @@ Accounts:
xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf0, msg0)
xcheckf(err, "writing deliver message to file")
err = accTest2.DeliverMessage(log, tx, &m0, mf0, false, false, false)
err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false)
xcheckf(err, "add message to account test2")
mf0name := mf0.Name()
@ -376,7 +375,7 @@ Accounts:
xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf1, msg1)
xcheckf(err, "writing deliver message to file")
err = accTest2.DeliverMessage(log, tx, &m1, mf1, false, false, false)
err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false)
xcheckf(err, "add message to account test2")
mf1name := mf1.Name()

11
http/autoconf.go
View file

@ -6,11 +6,12 @@ import (
"net/http"
"strings"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"rsc.io/qr"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp"
)
@ -55,7 +56,7 @@ var (
// User should create a DNS record: autoconfig.<domain> (CNAME or A).
// See https://wiki.mozilla.org/Thunderbird:Autoconfiguration:ConfigFileFormat
func autoconfHandle(w http.ResponseWriter, r *http.Request) {
log := xlog.WithContext(r.Context())
log := pkglog.WithContext(r.Context())
var addrDom string
defer func() {
@ -63,7 +64,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
}()
email := r.FormValue("emailaddress")
log.Debug("autoconfig request", mlog.Field("email", email))
log.Debug("autoconfig request", slog.String("email", email))
addr, err := smtp.ParseAddress(email)
if err != nil {
http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest)
@ -143,7 +144,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) {
//
// Thunderbird does understand autodiscover.
func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
log := xlog.WithContext(r.Context())
log := pkglog.WithContext(r.Context())
var addrDom string
defer func() {
@ -161,7 +162,7 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) {
return
}
log.Debug("autodiscover request", mlog.Field("email", req.Request.EmailAddress))
log.Debug("autodiscover request", slog.String("email", req.Request.EmailAddress))
addr, err := smtp.ParseAddress(req.Request.EmailAddress)
if err != nil {

22
http/gzcache.go
View file

@ -16,6 +16,8 @@ import (
"sync"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog"
)
@ -75,7 +77,7 @@ func loadStaticGzipCache(dir string, maxSize int64) {
os.MkdirAll(dir, 0700)
entries, err := os.ReadDir(dir)
if err != nil && !os.IsNotExist(err) {
xlog.Errorx("listing static gzip cache files", err, mlog.Field("dir", dir))
pkglog.Errorx("listing static gzip cache files", err, slog.String("dir", dir))
}
for _, e := range entries {
name := e.Name()
@ -111,9 +113,9 @@ func loadStaticGzipCache(dir string, maxSize int64) {
atime, err = statAtime(fi.Sys())
}
if err != nil {
xlog.Infox("removing unusable/unrecognized file in static gzip cache dir", err)
pkglog.Infox("removing unusable/unrecognized file in static gzip cache dir", err)
xerr := os.Remove(filepath.Join(dir, name))
xlog.Check(xerr, "removing unusable file in static gzip cache dir", mlog.Field("error", err), mlog.Field("dir", dir), mlog.Field("filename", name))
pkglog.Check(xerr, "removing unusable file in static gzip cache dir", slog.Any("error", err), slog.String("dir", dir), slog.String("filename", name))
continue
}
staticgzcache.paths[path] = gzfile{
@ -163,7 +165,7 @@ func (c *gzcache) evictPath(path string) {
c.unlink(gf.use)
c.size -= gf.gzsize
err := os.Remove(staticCachePath(c.dir, path, gf.mtime))
xlog.Check(err, "removing cached gzipped static file", mlog.Field("path", path))
pkglog.Check(err, "removing cached gzipped static file", slog.String("path", path))
}
// Open cached file for path, requiring it has mtime. If there is no usable cached
@ -189,7 +191,7 @@ func (c *gzcache) openPath(path string, mtime int64) (*os.File, int64) {
p := staticCachePath(c.dir, path, gf.mtime)
f, err := os.Open(p)
if err != nil {
xlog.Errorx("open static cached gzip file, removing from cache", err, mlog.Field("path", path))
pkglog.Errorx("open static cached gzip file, removing from cache", err, slog.String("path", path))
// Perhaps someone removed the file? Remove from cache, it will be recreated.
c.evictPath(path)
return nil, 0
@ -303,8 +305,8 @@ type staticgzcacheReplacer struct {
handled bool
}
func (w *staticgzcacheReplacer) logger() *mlog.Log {
return xlog.WithContext(w.r.Context())
func (w *staticgzcacheReplacer) logger() mlog.Log {
return pkglog.WithContext(w.r.Context())
}
// Header returns the header of the underlying ResponseWriter.
@ -353,7 +355,7 @@ func (w *staticgzcacheReplacer) WriteHeader(statusCode int) {
p := staticCachePath(staticgzcache.dir, w.uncomprPath, w.uncomprMtime.UnixNano())
ngzf, err := os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
if err != nil {
w.logger().Errorx("create new static gzip cache file", err, mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p))
w.logger().Errorx("create new static gzip cache file", err, slog.String("requestpath", w.uncomprPath), slog.String("fspath", p))
staticgzcache.abortPath(w.uncomprPath)
return
}
@ -361,9 +363,9 @@ func (w *staticgzcacheReplacer) WriteHeader(statusCode int) {
if ngzf != nil {
staticgzcache.abortPath(w.uncomprPath)
err := ngzf.Close()
w.logger().Check(err, "closing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p))
w.logger().Check(err, "closing failed static gzip cache file", slog.String("requestpath", w.uncomprPath), slog.String("fspath", p))
err = os.Remove(p)
w.logger().Check(err, "removing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p))
w.logger().Check(err, "removing failed static gzip cache file", slog.String("requestpath", w.uncomprPath), slog.String("fspath", p))
}
}()

10
http/mtasts.go
View file

@ -6,6 +6,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
@ -13,8 +15,8 @@ import (
)
func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
log := func() *mlog.Log {
return xlog.WithContext(r.Context())
log := func() mlog.Log {
return pkglog.WithContext(r.Context())
}
host := strings.ToLower(r.Host)
@ -30,7 +32,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
}
domain, err := dns.ParseDomain(host)
if err != nil {
log().Errorx("mtasts policy request: bad domain", err, mlog.Field("host", host))
log().Errorx("mtasts policy request: bad domain", err, slog.String("host", host))
http.NotFound(w, r)
return
}
@ -51,7 +53,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
}
d, err := dns.ParseDomain(s)
if err != nil {
log().Errorx("bad domain in mtasts config", err, mlog.Field("domain", s))
log().Errorx("bad domain in mtasts config", err, slog.String("domain", s))
http.Error(w, "500 - internal server error - invalid domain in configuration", http.StatusInternalServerError)
return
}

87
http/web.go
View file

@ -21,6 +21,7 @@ import (
_ "net/http/pprof"
"golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -37,7 +38,7 @@ import (
"github.com/mjl-/mox/webmail"
)
var xlog = mlog.New("http")
var pkglog = mlog.New("http", nil)
var (
// metricRequest tracks performance (time to write response header) of server.
@ -96,11 +97,11 @@ type loggingWriter struct {
Err error
WebsocketResponse bool // If this was a successful websocket connection with backend.
SizeFromClient, SizeToClient int64 // Websocket data.
Fields []mlog.Pair // Additional fields to log.
Attrs []slog.Attr // Additional fields to log.
}
func (w *loggingWriter) AddField(p mlog.Pair) {
w.Fields = append(w.Fields, p)
func (w *loggingWriter) AddAttr(a slog.Attr) {
w.Attrs = append(w.Attrs, a)
}
func (w *loggingWriter) Flush() {
@ -310,43 +311,43 @@ func (w *loggingWriter) Done() {
if err == nil {
err = w.R.Context().Err()
}
fields := []mlog.Pair{
mlog.Field("httpaccess", ""),
mlog.Field("handler", w.Handler),
mlog.Field("method", method),
mlog.Field("url", w.R.URL),
mlog.Field("host", w.R.Host),
mlog.Field("duration", time.Since(w.Start)),
mlog.Field("statuscode", w.StatusCode),
mlog.Field("proto", strings.ToLower(w.R.Proto)),
mlog.Field("remoteaddr", w.R.RemoteAddr),
mlog.Field("tlsinfo", tlsinfo),
mlog.Field("useragent", w.R.Header.Get("User-Agent")),
mlog.Field("referrr", w.R.Header.Get("Referrer")),
attrs := []slog.Attr{
slog.String("httpaccess", ""),
slog.String("handler", w.Handler),
slog.String("method", method),
slog.Any("url", w.R.URL),
slog.String("host", w.R.Host),
slog.Duration("duration", time.Since(w.Start)),
slog.Int("statuscode", w.StatusCode),
slog.String("proto", strings.ToLower(w.R.Proto)),
slog.Any("remoteaddr", w.R.RemoteAddr),
slog.String("tlsinfo", tlsinfo),
slog.String("useragent", w.R.Header.Get("User-Agent")),
slog.String("referrr", w.R.Header.Get("Referrer")),
}
if w.WebsocketRequest {
fields = append(fields,
mlog.Field("websocketrequest", true),
attrs = append(attrs,
slog.Bool("websocketrequest", true),
)
}
if w.WebsocketResponse {
fields = append(fields,
mlog.Field("websocket", true),
mlog.Field("sizetoclient", w.SizeToClient),
mlog.Field("sizefromclient", w.SizeFromClient),
attrs = append(attrs,
slog.Bool("websocket", true),
slog.Int64("sizetoclient", w.SizeToClient),
slog.Int64("sizefromclient", w.SizeFromClient),
)
} else if w.UncompressedSize > 0 {
fields = append(fields,
mlog.Field("size", w.Size),
mlog.Field("uncompressedsize", w.UncompressedSize),
attrs = append(attrs,
slog.Int64("size", w.Size),
slog.Int64("uncompressedsize", w.UncompressedSize),
)
} else {
fields = append(fields,
mlog.Field("size", w.Size),
attrs = append(attrs,
slog.Int64("size", w.Size),
)
}
fields = append(fields, w.Fields...)
xlog.WithContext(w.R.Context()).Debugx("http request", err, fields...)
attrs = append(attrs, w.Attrs...)
pkglog.WithContext(w.R.Context()).Debugx("http request", err, attrs...)
}
// Set some http headers that should prevent potential abuse. Better safe than sorry.
@ -405,9 +406,9 @@ func (s *serve) ServeHTTP(xw http.ResponseWriter, r *http.Request) {
// Rate limiting as early as possible.
ipstr, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
xlog.Debugx("split host:port client remoteaddr", err, mlog.Field("remoteaddr", r.RemoteAddr))
pkglog.Debugx("split host:port client remoteaddr", err, slog.Any("remoteaddr", r.RemoteAddr))
} else if ip := net.ParseIP(ipstr); ip == nil {
xlog.Debug("parsing ip for client remoteaddr", mlog.Field("remoteaddr", r.RemoteAddr))
pkglog.Debug("parsing ip for client remoteaddr", slog.Any("remoteaddr", r.RemoteAddr))
} else if !limiterConnectionrate.Add(ip, now, 1) {
method := metricHTTPMethod(r.Method)
proto := "http"
@ -649,7 +650,7 @@ func Listen() {
// Importing net/http/pprof registers handlers on the default serve mux.
port := config.Port(l.PprofHTTP.Port, 8011)
if _, ok := portServe[port]; ok {
xlog.Fatal("cannot serve pprof on same endpoint as other http services")
pkglog.Fatal("cannot serve pprof on same endpoint as other http services")
}
srv := &serve{[]string{"pprof-http"}, nil, nil, false}
portServe[port] = srv
@ -686,7 +687,7 @@ func Listen() {
// presence of TLS certificates for.
for _, name := range mox.Conf.Domains() {
if dom, err := dns.ParseDomain(name); err != nil {
xlog.Errorx("parsing domain from config", err)
pkglog.Errorx("parsing domain from config", err)
} else if d, _ := mox.Conf.Domain(dom); d.DMARC != nil && d.DMARC.Domain != "" && d.DMARC.DNSDomain != dom {
// Do not gather autoconfig name if this domain is configured to process reports
// for domains hosted elsewhere.
@ -695,7 +696,7 @@ func Listen() {
autoconfdom, err := dns.ParseDomain("autoconfig." + name)
if err != nil {
xlog.Errorx("parsing domain from config for autoconfig", err)
pkglog.Errorx("parsing domain from config for autoconfig", err)
} else {
hosts[autoconfdom] = struct{}{}
}
@ -745,20 +746,20 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st
if tlsConfig == nil {
protocol = "http"
if os.Getuid() == 0 {
xlog.Print("http listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr))
pkglog.Print("http listener", slog.String("name", name), slog.String("kinds", strings.Join(kinds, ",")), slog.String("address", addr))
}
ln, err = mox.Listen(mox.Network(ip), addr)
if err != nil {
xlog.Fatalx("http: listen", err, mlog.Field("addr", addr))
pkglog.Fatalx("http: listen", err, slog.Any("addr", addr))
}
} else {
protocol = "https"
if os.Getuid() == 0 {
xlog.Print("https listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr))
pkglog.Print("https listener", slog.String("name", name), slog.String("kinds", strings.Join(kinds, ",")), slog.String("address", addr))
}
ln, err = mox.Listen(mox.Network(ip), addr)
if err != nil {
xlog.Fatalx("https: listen", err, mlog.Field("addr", addr))
pkglog.Fatalx("https: listen", err, slog.String("addr", addr))
}
ln = tls.NewListener(ln, tlsConfig)
}
@ -768,11 +769,11 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st
TLSConfig: tlsConfig,
ReadHeaderTimeout: 30 * time.Second,
IdleTimeout: 65 * time.Second, // Chrome closes connections after 60 seconds, firefox after 115 seconds.
ErrorLog: golog.New(mlog.ErrWriter(xlog.Fields(mlog.Field("pkg", "net/http")), mlog.LevelInfo, protocol+" error"), "", 0),
ErrorLog: golog.New(mlog.LogWriter(pkglog.With(slog.String("pkg", "net/http")), slog.LevelInfo, protocol+" error"), "", 0),
}
serve := func() {
err := server.Serve(ln)
xlog.Fatalx(protocol+": serve", err)
pkglog.Fatalx(protocol+": serve", err)
}
servers = append(servers, serve)
}
@ -815,9 +816,9 @@ func Serve() {
SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256},
SupportedVersions: []uint16{tls.VersionTLS13},
}
xlog.Print("ensuring certificate availability", mlog.Field("hostname", host))
pkglog.Print("ensuring certificate availability", slog.Any("hostname", host))
if _, err := m.Manager.GetCertificate(hello); err != nil {
xlog.Errorx("requesting automatic certificate", err, mlog.Field("hostname", host))
pkglog.Errorx("requesting automatic certificate", err, slog.Any("hostname", host))
}
}
}

34
http/webserver.go
View file

@ -25,6 +25,8 @@ import (
"syscall"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
@ -149,8 +151,8 @@ table > tbody > tr:nth-child(odd) { background-color: #f8f8f8; }
// file is returned. Otherwise, for directories with ListFiles configured, a
// directory listing is returned.
func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *http.Request) (handled bool) {
log := func() *mlog.Log {
return xlog.WithContext(r.Context())
log := func() mlog.Log {
return pkglog.WithContext(r.Context())
}
if r.Method != "GET" && r.Method != "HEAD" {
if h.ContinueNotFound {
@ -217,7 +219,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
var ifi os.FileInfo
ifi, err = index.Stat()
if err != nil {
log().Errorx("stat index.html in directory we cannot list", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
log().Errorx("stat index.html in directory we cannot list", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true
}
@ -228,7 +230,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
http.Error(w, "403 - permission denied", http.StatusForbidden)
return true
}
log().Errorx("open file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
log().Errorx("open file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true
}
@ -236,7 +238,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
fi, err := f.Stat()
if err != nil {
log().Errorx("stat file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
log().Errorx("stat file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true
}
@ -274,7 +276,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
}
}
if !os.IsNotExist(err) {
log().Errorx("stat for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
log().Errorx("stat for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true
}
@ -315,7 +317,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *
if err == io.EOF {
break
} else if err != nil {
log().Errorx("reading directory for file listing", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
log().Errorx("reading directory for file listing", err, slog.Any("url", r.URL), slog.String("fspath", fspath))
http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError)
return true
}
@ -398,8 +400,8 @@ func HandleRedirect(h *config.WebRedirect, w http.ResponseWriter, r *http.Reques
// connections by monitoring the websocket handshake and then just passing along the
// websocket frames.
func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) {
log := func() *mlog.Log {
return xlog.WithContext(r.Context())
log := func() mlog.Log {
return pkglog.WithContext(r.Context())
}
xr := *r
@ -459,13 +461,13 @@ func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request,
// ReverseProxy will append any remaining path to the configured target URL.
proxy := httputil.NewSingleHostReverseProxy(h.TargetURL)
proxy.FlushInterval = time.Duration(-1) // Flush after each write.
proxy.ErrorLog = golog.New(mlog.ErrWriter(mlog.New("net/http/httputil").WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0)
proxy.ErrorLog = golog.New(mlog.LogWriter(mlog.New("net/http/httputil", nil).WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0)
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
if errors.Is(err, context.Canceled) {
log().Debugx("forwarding request to backend webserver", err, mlog.Field("url", r.URL))
log().Debugx("forwarding request to backend webserver", err, slog.Any("url", r.URL))
return
}
log().Errorx("forwarding request to backend webserver", err, mlog.Field("url", r.URL))
log().Errorx("forwarding request to backend webserver", err, slog.Any("url", r.URL))
if os.IsTimeout(err) {
http.Error(w, "504 - gateway timeout"+recvid(r), http.StatusGatewayTimeout)
} else {
@ -493,8 +495,8 @@ var errNotImplemented = errors.New("functionality not yet implemented")
// work for little benefit. Besides, the whole point of websockets is to exchange
// bytes without HTTP being in the way, so let's do that.
func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) {
log := func() *mlog.Log {
return xlog.WithContext(r.Context())
log := func() mlog.Log {
return pkglog.WithContext(r.Context())
}
lw := w.(*loggingWriter)
@ -658,8 +660,8 @@ func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Reque
}
func websocketTransact(ctx context.Context, targetURL *url.URL, r *http.Request) (rresp *http.Response, rconn net.Conn, rerr error) {
log := func() *mlog.Log {
return xlog.WithContext(r.Context())
log := func() mlog.Log {
return pkglog.WithContext(r.Context())
}
// Dial the backend, possibly doing TLS. We assume the net/http DefaultTransport is

6
imapserver/fetch.go
View file

@ -12,11 +12,11 @@ import (
"strings"
"golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
@ -233,7 +233,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
for _, uid := range uids {
cmd.uid = uid
mlog.Field("processing uid", mlog.Field("uid", uid))
cmd.conn.log.Debug("processing uid", slog.Any("uid", uid))
cmd.process(atts)
}
@ -326,7 +326,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
cmd.expungeIssued = true
return
}
cmd.conn.log.Infox("processing fetch attribute", err, mlog.Field("uid", cmd.uid))
cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid))
xuserErrorf("processing fetch attribute: %v", err)
}()

6
imapserver/fuzz_test.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store"
)
@ -58,17 +59,18 @@ func FuzzServer(f *testing.F) {
f.Add(tag + cmd)
}
log := mlog.New("imapserver", nil)
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/imapserverfuzz/mox.conf")
mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
os.RemoveAll(dataDir)
acc, err := store.OpenAccount("mjl")
acc, err := store.OpenAccount(log, "mjl")
if err != nil {
f.Fatalf("open account: %v", err)
}
defer acc.Close()
err = acc.SetPassword("testtest")
err = acc.SetPassword(log, "testtest")
if err != nil {
f.Fatalf("set password: %v", err)
}

4
imapserver/parse.go
View file

@ -8,7 +8,7 @@ import (
"strings"
"time"
"github.com/mjl-/mox/mlog"
"golang.org/x/exp/slog"
)
var (
@ -402,7 +402,7 @@ func (p *parser) xmailbox() string {
if !p.conn.enabled[capIMAP4rev2] {
ns, err := utf7decode(s)
if err != nil {
p.conn.log.Infox("decoding utf7 or mailbox name", err, mlog.Field("name", s))
p.conn.log.Infox("decoding utf7 or mailbox name", err, slog.String("name", s))
} else {
s = ns
}

9
imapserver/search.go
View file

@ -5,10 +5,11 @@ import (
"net/textproto"
"strings"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/store"
)
@ -393,7 +394,7 @@ func (s *search) match0(sk searchKey) bool {
lower := strings.ToLower(value)
h, err := s.p.Header()
if err != nil {
c.log.Debugx("parsing message header", err, mlog.Field("uid", s.uid))
c.log.Debugx("parsing message header", err, slog.Any("uid", s.uid))
return false
}
for _, v := range h.Values(field) {
@ -517,7 +518,7 @@ func (s *search) match0(sk searchKey) bool {
}
if s.p == nil {
c.log.Info("missing parsed message, not matching", mlog.Field("uid", s.uid))
c.log.Info("missing parsed message, not matching", slog.Any("uid", s.uid))
return false
}
@ -546,7 +547,7 @@ func (s *search) match0(sk searchKey) bool {
lower := strings.ToLower(sk.astring)
h, err := s.p.Header()
if err != nil {
c.log.Errorx("parsing header for search", err, mlog.Field("uid", s.uid))
c.log.Errorx("parsing header for search", err, slog.Any("uid", s.uid))
return false
}
k := textproto.CanonicalMIMEHeaderKey(sk.headerField)

94
imapserver/server.go
View file

@ -56,10 +56,12 @@ import (
"runtime/debug"
"sort"
"strings"
"sync"
"time"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -78,10 +80,6 @@ import (
"github.com/mjl-/mox/store"
)
// Most logging should be done through conn.log* functions.
// Only use imaplog in contexts without connection.
var xlog = mlog.New("imapserver")
var (
metricIMAPConnection = promauto.NewCounterVec(
prometheus.CounterOpts{
@ -180,7 +178,7 @@ type conn struct {
cmdMetric string // Currently executing, for metrics.
cmdStart time.Time
ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid.
log *mlog.Log
log mlog.Log
enabled map[capability]bool // All upper-case.
// Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with
@ -338,14 +336,15 @@ func Listen() {
var servers []func()
func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noRequireSTARTTLS bool) {
log := mlog.New("imapserver", nil)
addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
if os.Getuid() == 0 {
xlog.Print("listening for imap", mlog.Field("listener", listenerName), mlog.Field("addr", addr), mlog.Field("protocol", protocol))
log.Print("listening for imap", slog.String("listener", listenerName), slog.String("addr", addr), slog.String("protocol", protocol))
}
network := mox.Network(ip)
ln, err := mox.Listen(network, addr)
if err != nil {
xlog.Fatalx("imap: listen for imap", err, mlog.Field("protocol", protocol), mlog.Field("listener", listenerName))
log.Fatalx("imap: listen for imap", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
}
if xtls {
ln = tls.NewListener(ln, tlsConfig)
@ -355,7 +354,7 @@ func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config,
for {
conn, err := ln.Accept()
if err != nil {
xlog.Infox("imap: accept", err, mlog.Field("protocol", protocol), mlog.Field("listener", listenerName))
log.Infox("imap: accept", err, slog.String("protocol", protocol), slog.String("listener", listenerName))
continue
}
@ -441,7 +440,7 @@ func (c *conn) Write(buf []byte) (int, error) {
return n, nil
}
func (c *conn) xtrace(level mlog.Level) func() {
func (c *conn) xtrace(level slog.Level) func() {
c.xflush()
c.tr.SetTrace(level)
c.tw.SetTrace(level)
@ -469,7 +468,7 @@ func (c *conn) readline0() (string, error) {
err := c.conn.SetReadDeadline(time.Now().Add(d))
c.log.Check(err, "setting read deadline")
line, err := bufpool.Readline(c.br)
line, err := bufpool.Readline(c.log, c.br)
if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
return "", fmt.Errorf("%s (%w)", err, errProtocol)
} else if err != nil {
@ -629,15 +628,18 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
cmd: "(greeting)",
cmdStart: time.Now(),
}
c.log = xlog.MoreFields(func() []mlog.Pair {
var logmutex sync.Mutex
c.log = mlog.New("imapserver", nil).WithFunc(func() []slog.Attr {
logmutex.Lock()
defer logmutex.Unlock()
now := time.Now()
l := []mlog.Pair{
mlog.Field("cid", c.cid),
mlog.Field("delta", now.Sub(c.lastlog)),
l := []slog.Attr{
slog.Int64("cid", c.cid),
slog.Duration("delta", now.Sub(c.lastlog)),
}
c.lastlog = now
if c.username != "" {
l = append(l, mlog.Field("username", c.username))
l = append(l, slog.String("username", c.username))
}
return l
})
@ -662,7 +664,7 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
}
}
c.log.Info("new connection", mlog.Field("remote", c.conn.RemoteAddr()), mlog.Field("local", c.conn.LocalAddr()), mlog.Field("tls", xtls), mlog.Field("listener", listenerName))
c.log.Info("new connection", slog.Any("remote", c.conn.RemoteAddr()), slog.Any("local", c.conn.LocalAddr()), slog.Bool("tls", xtls), slog.String("listener", listenerName))
defer func() {
c.conn.Close()
@ -681,7 +683,7 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
} else if err, ok := x.(error); ok && isClosed(err) {
c.log.Infox("connection closed", err)
} else {
c.log.Error("unhandled panic", mlog.Field("err", x))
c.log.Error("unhandled panic", slog.Any("err", x))
debug.PrintStack()
metrics.PanicInc(metrics.Imapserver)
}
@ -703,13 +705,13 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
// If remote IP/network resulted in too many authentication failures, refuse to serve.
if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
metrics.AuthenticationRatelimitedInc("imap")
c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP))
c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
c.writelinef("* BYE too many auth failures")
return
}
if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP))
c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
c.writelinef("* BYE too many open connections from your ip or network")
return
}
@ -744,9 +746,9 @@ func (c *conn) command() {
metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
}()
logFields := []mlog.Pair{
mlog.Field("cmd", c.cmd),
mlog.Field("duration", time.Since(c.cmdStart)),
logFields := []slog.Attr{
slog.String("cmd", c.cmd),
slog.Duration("duration", time.Since(c.cmdStart)),
}
c.cmd = ""
@ -761,7 +763,7 @@ func (c *conn) command() {
}
err, ok := x.(error)
if !ok {
c.log.Error("imap command panic", append([]mlog.Pair{mlog.Field("panic", x)}, logFields...)...)
c.log.Error("imap command panic", append([]slog.Attr{slog.Any("panic", x)}, logFields...)...)
result = "panic"
panic(x)
}
@ -786,7 +788,7 @@ func (c *conn) command() {
panic(errIO)
}
c.log.Debugx("imap command syntax error", sxerr.err, logFields...)
c.log.Info("imap syntax error", mlog.Field("lastline", c.lastLine))
c.log.Info("imap syntax error", slog.String("lastline", c.lastLine))
fatal := strings.HasSuffix(c.lastLine, "+}")
if fatal {
err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
@ -865,7 +867,7 @@ func (c *conn) broadcast(changes []store.Change) {
if len(changes) == 0 {
return
}
c.log.Debug("broadcast changes", mlog.Field("changes", changes))
c.log.Debug("broadcast changes", slog.Any("changes", changes))
c.comm.Broadcast(changes)
}
@ -1184,7 +1186,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute))
c.log.Check(err, "setting write deadline")
c.log.Debug("applying changes", mlog.Field("changes", changes))
c.log.Debug("applying changes", slog.Any("changes", changes))
// Only keep changes for the selected mailbox, and changes that are always relevant.
var n []store.Change
@ -1404,7 +1406,7 @@ func (c *conn) cmdID(tag, cmd string, p *parser) {
p.xempty()
// We just log the client id.
c.log.Info("client id", mlog.Field("params", params))
c.log.Info("client id", slog.Any("params", params))
// Response syntax: ../rfc/2971:243
// We send our name and version. ../rfc/2971:193
@ -1448,7 +1450,7 @@ func (c *conn) cmdStarttls(tag, cmd string, p *parser) {
}
cancel()
tlsversion, ciphersuite := mox.TLSInfo(tlsConn)
c.log.Debug("tls server handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite))
c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite))
c.conn = tlsConn
c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn)
@ -1560,11 +1562,11 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role")
}
acc, err := store.OpenEmailAuth(authc, password)
acc, err := store.OpenEmailAuth(c.log, authc, password)
if err != nil {
if errors.Is(err, store.ErrUnknownCredentials) {
authResult = "badcreds"
c.log.Info("authentication failed", mlog.Field("username", authc))
c.log.Info("authentication failed", slog.String("username", authc))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
}
xusercodeErrorf("", "error")
@ -1588,11 +1590,11 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xsyntaxErrorf("malformed cram-md5 response")
}
addr := t[0]
c.log.Debug("cram-md5 auth", mlog.Field("address", addr))
acc, _, err := store.OpenEmail(addr)
c.log.Debug("cram-md5 auth", slog.String("address", addr))
acc, _, err := store.OpenEmail(c.log, addr)
if err != nil {
if errors.Is(err, store.ErrUnknownCredentials) {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
}
xserverErrorf("looking up address: %v", err)
@ -1608,7 +1610,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
password, err := bstore.QueryTx[store.Password](tx).Get()
if err == bstore.ErrAbsent {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
}
if err != nil {
@ -1622,8 +1624,8 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xcheckf(err, "tx read")
})
if ipadhash == nil || opadhash == nil {
c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", mlog.Field("username", addr))
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", addr))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
}
@ -1632,7 +1634,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
opadhash.Write(ipadhash.Sum(nil))
digest := fmt.Sprintf("%x", opadhash.Sum(nil))
if digest != t[1] {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
}
@ -1659,8 +1661,8 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
if err != nil {
xsyntaxErrorf("starting scram: %s", err)
}
c.log.Debug("scram auth", mlog.Field("authentication", ss.Authentication))
acc, _, err := store.OpenEmail(ss.Authentication)
c.log.Debug("scram auth", slog.String("authentication", ss.Authentication))
acc, _, err := store.OpenEmail(c.log, ss.Authentication)
if err != nil {
// todo: we could continue scram with a generated salt, deterministically generated
// from the username. that way we don't have to store anything but attackers cannot
@ -1686,7 +1688,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
xscram = password.SCRAMSHA256
}
if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) {
c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", mlog.Field("address", ss.Authentication))
c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("address", ss.Authentication))
xuserErrorf("scram not possible")
}
xcheckf(err, "fetching credentials")
@ -1706,7 +1708,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
c.readline(false) // Should be "*" for cancellation.
if errors.Is(err, scram.ErrInvalidProof) {
authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
}
xuserErrorf("server final: %w", err)
@ -1770,13 +1772,13 @@ func (c *conn) cmdLogin(tag, cmd string, p *parser) {
}
}()
acc, err := store.OpenEmailAuth(userid, password)
acc, err := store.OpenEmailAuth(c.log, userid, password)
if err != nil {
authResult = "badcreds"
var code string
if errors.Is(err, store.ErrUnknownCredentials) {
code = "AUTHENTICATIONFAILED"
c.log.Info("failed authentication attempt", mlog.Field("username", userid), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", userid), slog.Any("remote", c.remoteIP))
}
xusercodeErrorf(code, "login failed")
}
@ -2251,7 +2253,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
for _, mID := range removeMessageIDs {
p := c.account.MessagePath(mID)
err := os.Remove(p)
c.log.Check(err, "removing message file for mailbox delete", mlog.Field("path", p))
c.log.Check(err, "removing message file for mailbox delete", slog.String("path", p))
}
c.ok(tag, cmd)
@ -2674,7 +2676,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
}
// Read the message into a temporary file.
msgFile, err := store.CreateMessageTemp("imap-append")
msgFile, err := store.CreateMessageTemp(c.log, "imap-append")
xcheckf(err, "creating temp file for message")
defer func() {
p := msgFile.Name()
@ -3273,7 +3275,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
}
for dir := range syncDirs {
err := moxio.SyncDir(dir)
err := moxio.SyncDir(c.log, dir)
xcheckf(err, "sync directory")
}

6
imapserver/server_test.go
View file

@ -17,12 +17,14 @@ import (
"time"
"github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
)
var ctxbg = context.Background()
var pkglog = mlog.New("imapserver", nil)
func init() {
sanityChecks = true
@ -341,10 +343,10 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf")
mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl")
acc, err := store.OpenAccount(pkglog, "mjl")
tcheck(t, err, "open account")
if first {
err = acc.SetPassword("testtest")
err = acc.SetPassword(pkglog, "testtest")
tcheck(t, err, "set password")
}
switchStop := func() {}

29
import.go
View file

@ -16,11 +16,11 @@ import (
"time"
"golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store"
)
@ -119,10 +119,9 @@ func xcmdXImport(mbox bool, c *cmd) {
}
defer store.Switchboard()()
xlog := mlog.New("import")
cconn, sconn := net.Pipe()
clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: xlog}
serverctl := ctl{conn: sconn, r: bufio.NewReader(sconn), log: xlog}
clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: c.log}
serverctl := ctl{conn: sconn, r: bufio.NewReader(sconn), log: c.log}
go servectlcmd(context.Background(), &serverctl, func() {})
ctlcmdImport(&clientctl, mbox, account, args[1], args[2])
@ -177,7 +176,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
if mbox {
kind = "mbox"
}
ctl.log.Info("importing messages", mlog.Field("kind", kind), mlog.Field("account", account), mlog.Field("mailbox", mailbox), mlog.Field("source", src))
ctl.log.Info("importing messages", slog.String("kind", kind), slog.String("account", account), slog.String("mailbox", mailbox), slog.String("source", src))
var err error
var mboxf *os.File
@ -186,7 +185,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
// Open account, creating a database file if it doesn't exist yet. It must be known
// in the configuration file.
a, err := store.OpenAccount(account)
a, err := store.OpenAccount(ctl.log, account)
ctl.xcheck(err, "opening account")
defer func() {
if a != nil {
@ -222,13 +221,13 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
if mbox {
mboxf, err = os.Open(src)
ctl.xcheck(err, "open mbox file")
msgreader = store.NewMboxReader(store.CreateMessageTemp, src, mboxf, ctl.log)
msgreader = store.NewMboxReader(ctl.log, store.CreateMessageTemp, src, mboxf)
} else {
mdnewf, err = os.Open(filepath.Join(src, "new"))
ctl.xcheck(err, "open subdir new of maildir")
mdcurf, err = os.Open(filepath.Join(src, "cur"))
ctl.xcheck(err, "open subdir cur of maildir")
msgreader = store.NewMaildirReader(store.CreateMessageTemp, mdnewf, mdcurf, ctl.log)
msgreader = store.NewMaildirReader(ctl.log, store.CreateMessageTemp, mdnewf, mdcurf)
}
tx, err := a.DB.Begin(ctx, true)
@ -253,7 +252,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
}
if x != ctl.x {
ctl.log.Error("import error", mlog.Field("panic", fmt.Errorf("%v", x)))
ctl.log.Error("import error", slog.String("panic", fmt.Sprintf("%v", x)))
debug.PrintStack()
metrics.PanicInc(metrics.Import)
} else {
@ -263,7 +262,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
for _, id := range deliveredIDs {
p := a.MessagePath(id)
err := os.Remove(p)
ctl.log.Check(err, "closing message file after import error", mlog.Field("path", p))
ctl.log.Check(err, "closing message file after import error", slog.String("path", p))
}
ctl.xerror(fmt.Sprintf("import error: %v", x))
@ -282,7 +281,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
err := a.DeliverMessage(ctl.log, tx, m, mf, sync, notrain, nothreads)
ctl.xcheck(err, "delivering message")
deliveredIDs = append(deliveredIDs, m.ID)
ctl.log.Debug("delivered message", mlog.Field("id", m.ID))
ctl.log.Debug("delivered message", slog.Int64("id", m.ID))
changes = append(changes, m.ChangeAddUID())
}
@ -319,9 +318,9 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
mb.Add(m.MailboxCounts())
// Parse message and store parsed information for later fast retrieval.
p, err := message.EnsurePart(ctl.log, false, msgf, m.Size)
p, err := message.EnsurePart(ctl.log.Logger, false, msgf, m.Size)
if err != nil {
ctl.log.Infox("parsing message, continuing", err, mlog.Field("path", origPath))
ctl.log.Infox("parsing message, continuing", err, slog.String("path", origPath))
}
m.ParsedBuf, err = json.Marshal(p)
ctl.xcheck(err, "marshal parsed message structure")
@ -345,7 +344,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
m.JunkFlagsForMailbox(mb, conf)
if jf != nil && m.NeedsTraining() {
if words, err := jf.ParseMessage(p); err != nil {
ctl.log.Infox("parsing message for updating junk filter", err, mlog.Field("parse", ""), mlog.Field("path", origPath))
ctl.log.Infox("parsing message for updating junk filter", err, slog.String("parse", ""), slog.String("path", origPath))
} else {
err = jf.Train(ctx, !m.Junk, words)
ctl.xcheck(err, "training junk filter")
@ -407,7 +406,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
err = tx.Commit()
ctl.xcheck(err, "commit")
tx = nil
ctl.log.Info("delivered messages through import", mlog.Field("count", len(deliveredIDs)))
ctl.log.Info("delivered messages through import", slog.Int("count", len(deliveredIDs)))
deliveredIDs = nil
store.BroadcastChanges(a, changes)

28
integration_test.go
View file

@ -14,6 +14,8 @@ import (
"testing"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/mlog"
@ -30,7 +32,7 @@ func tcheck(t *testing.T, err error, errmsg string) {
}
func TestDeliver(t *testing.T) {
xlog := mlog.New("integration")
log := mlog.New("integration", nil)
mlog.Logfmt = true
hostname, err := os.Hostname()
@ -129,7 +131,7 @@ This is the message.
`, mailfrom, rcptto)
msg = strings.ReplaceAll(msg, "\n", "\r\n")
auth := []sasl.Client{sasl.NewClientPlain(mailfrom, password)}
c, err := smtpclient.New(mox.Context, xlog, conn, smtpclient.TLSSkip, false, ourHostname, dns.Domain{ASCII: desthost}, smtpclient.Opts{Auth: auth})
c, err := smtpclient.New(mox.Context, log.Logger, conn, smtpclient.TLSSkip, false, ourHostname, dns.Domain{ASCII: desthost}, smtpclient.Opts{Auth: auth})
tcheck(t, err, "smtp hello")
err = c.Deliver(mox.Context, mailfrom, rcptto, int64(len(msg)), strings.NewReader(msg), false, false, false)
tcheck(t, err, "deliver with smtp")
@ -142,35 +144,35 @@ This is the message.
tcheck(t, err, "dial submission")
defer conn.Close()
xlog.Print("submitting email to moxacmepebble, waiting for imap notification at moxmail2")
log.Print("submitting email to moxacmepebble, waiting for imap notification at moxmail2")
t0 := time.Now()
deliver(true, true, "moxmail2.mox2.example:993", "moxtest2@mox2.example", "accountpass4321", func() {
submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "moxtest2@mox2.example")
})
xlog.Print("success", mlog.Field("duration", time.Since(t0)))
log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to moxmail2, waiting for imap notification at moxacmepebble")
log.Print("submitting email to moxmail2, waiting for imap notification at moxacmepebble")
t0 = time.Now()
deliver(true, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() {
submit(true, "moxtest2@mox2.example", "accountpass4321", "moxmail2.mox2.example:465", "moxtest1@mox1.example")
})
xlog.Print("success", mlog.Field("duration", time.Since(t0)))
log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to postfix, waiting for imap notification at moxacmepebble")
log.Print("submitting email to postfix, waiting for imap notification at moxacmepebble")
t0 = time.Now()
deliver(false, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() {
submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "root@postfix.example")
})
xlog.Print("success", mlog.Field("duration", time.Since(t0)))
log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to localserve")
log.Print("submitting email to localserve")
t0 = time.Now()
deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() {
submit(false, "mox@localhost", "moxmoxmox", "localserve.mox1.example:1587", "moxtest1@mox1.example")
})
xlog.Print("success", mlog.Field("duration", time.Since(t0)))
log.Print("success", slog.Duration("duration", time.Since(t0)))
xlog.Print("submitting email to localserve")
log.Print("submitting email to localserve")
t0 = time.Now()
deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() {
cmd := exec.Command("go", "run", ".", "sendmail", "mox@localhost")
@ -182,8 +184,8 @@ a message.
var out strings.Builder
cmd.Stdout = &out
err := cmd.Run()
xlog.Print("sendmail", mlog.Field("output", out.String()))
log.Print("sendmail", slog.String("output", out.String()))
tcheck(t, err, "sendmail")
})
xlog.Print("success", mlog.Field("duration", time.Since(t0)))
log.Print("success", slog.Any("duration", time.Since(t0)))
}

6
iprev/iprev.go
View file

@ -9,6 +9,8 @@ import (
"net"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -16,7 +18,7 @@ import (
"github.com/mjl-/mox/mlog"
)
var xlog = mlog.New("iprev")
var xlog = mlog.New("iprev", nil)
var (
metricIPRev = promauto.NewHistogramVec(
@ -61,7 +63,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, ip net.IP) (rstatus Stat
start := time.Now()
defer func() {
metricIPRev.WithLabelValues(string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("iprev lookup result", rerr, mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("duration", time.Since(start)))
log.Debugx("iprev lookup result", rerr, slog.Any("ip", ip), slog.Any("status", rstatus), slog.Duration("duration", time.Since(start)))
}()
revNames, result, revErr := dns.WithPackage(resolver, "iprev").LookupAddr(ctx, ip.String())

16
junk.go
View file

@ -93,7 +93,7 @@ func cmdJunkTrain(c *cmd) {
}
a.SetLogLevel()
f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath))
f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath))
defer func() {
if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err)
@ -122,7 +122,7 @@ func cmdJunkCheck(c *cmd) {
}
a.SetLogLevel()
f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false))
f := must(junk.OpenFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath, false))
defer func() {
if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err)
@ -146,7 +146,7 @@ func cmdJunkTest(c *cmd) {
}
a.SetLogLevel()
f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false))
f := must(junk.OpenFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath, false))
defer func() {
if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err)
@ -202,7 +202,7 @@ messages are shuffled, with optional random seed.`
}
a.SetLogLevel()
f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath))
f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath))
defer func() {
if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err)
@ -293,7 +293,7 @@ func cmdJunkPlay(c *cmd) {
}
a.SetLogLevel()
f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath))
f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath))
defer func() {
if err := f.Close(); err != nil {
log.Printf("closing junk filter: %v", err)
@ -310,8 +310,6 @@ func cmdJunkPlay(c *cmd) {
var nbad, nnodate, nham, nspam, nsent int
jlog := mlog.New("junkplay")
scanDir := func(dir string, ham, sent bool) {
for _, name := range listDir(dir) {
path := filepath.Join(dir, name)
@ -319,7 +317,7 @@ func cmdJunkPlay(c *cmd) {
xcheckf(err, "open %q", path)
fi, err := mf.Stat()
xcheckf(err, "stat %q", path)
p, err := message.EnsurePart(jlog, false, mf, fi.Size())
p, err := message.EnsurePart(c.log.Logger, false, mf, fi.Size())
if err != nil {
nbad++
if err := mf.Close(); err != nil {
@ -399,7 +397,7 @@ func cmdJunkPlay(c *cmd) {
}()
fi, err := mf.Stat()
xcheckf(err, "stat %q", path)
p, err := message.EnsurePart(jlog, false, mf, fi.Size())
p, err := message.EnsurePart(c.log.Logger, false, mf, fi.Size())
if err != nil {
log.Printf("bad sent message %q: %s", path, err)
return

61
junk/filter.go
View file

@ -21,6 +21,8 @@ import (
"sort"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
@ -28,8 +30,6 @@ import (
)
var (
xlog = mlog.New("junk")
// errBadContentType = errors.New("bad content-type") // sure sign of spam, todo: use this error
errClosed = errors.New("filter is closed")
)
@ -62,7 +62,7 @@ var DBTypes = []any{wordscore{}} // Stored in DB.
type Filter struct {
Params
log *mlog.Log // For logging cid.
log mlog.Log // For logging cid.
closed bool
modified bool // Whether any modifications are pending. Cleared by Save.
hams, spams uint32 // Message count, stored in db under word "-".
@ -112,7 +112,7 @@ func (f *Filter) Close() error {
return err
}
func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) {
func OpenFilter(ctx context.Context, log mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) {
var bloom *Bloom
if loadBloom {
var err error
@ -160,7 +160,7 @@ func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloom
// filter is marked as new until the first save, will be done automatically if
// TrainDirs is called. If the bloom and/or database files exist, an error is
// returned.
func NewFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) {
func NewFilter(ctx context.Context, log mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) {
var err error
if _, err := os.Stat(bloomPath); err == nil {
return nil, fmt.Errorf("bloom filter already exists on disk: %s", bloomPath)
@ -220,7 +220,7 @@ func openBloom(path string) (*Bloom, error) {
return NewBloom(buf, bloomK)
}
func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr error) {
func newDB(ctx context.Context, log mlog.Log, path string) (db *bstore.DB, rerr error) {
// Remove any existing files.
os.Remove(path)
@ -272,7 +272,7 @@ func (f *Filter) Save() error {
return words[i] < words[j]
})
f.log.Debug("inserting words in junkfilter db", mlog.Field("words", len(f.changed)))
f.log.Debug("inserting words in junkfilter db", slog.Any("words", len(f.changed)))
// start := time.Now()
if f.isNew {
if err := f.db.HintAppend(true, wordscore{}); err != nil {
@ -318,7 +318,7 @@ func (f *Filter) Save() error {
f.changed = map[string]word{}
f.modified = false
f.isNew = false
// f.log.Info("wrote filter to db", mlog.Field("duration", time.Since(start)))
// f.log.Info("wrote filter to db", slog.Any("duration", time.Since(start)))
return nil
}
@ -378,7 +378,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (
expect[w] = struct{}{}
}
if len(unknowns) > 0 {
f.log.Debug("unknown words in bloom filter, showing max 50", mlog.Field("words", unknowns), mlog.Field("totalunknown", totalUnknown), mlog.Field("totalwords", len(words)))
f.log.Debug("unknown words in bloom filter, showing max 50", slog.Any("words", unknowns), slog.Any("totalunknown", totalUnknown), slog.Any("totalwords", len(words)))
}
// Fetch words from database.
@ -391,7 +391,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (
delete(expect, w)
f.cache[w] = c
}
f.log.Debug("unknown words in db", mlog.Field("words", expect), mlog.Field("totalunknown", len(expect)), mlog.Field("totalwords", len(words)))
f.log.Debug("unknown words in db", slog.Any("words", expect), slog.Any("totalunknown", len(expect)), slog.Any("totalwords", len(words)))
}
for w := range words {
@ -474,7 +474,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (
eta += math.Log(1-x.R) - math.Log(x.R)
}
f.log.Debug("top words", mlog.Field("hams", topHam), mlog.Field("spams", topSpam))
f.log.Debug("top words", slog.Any("hams", topHam), slog.Any("spams", topSpam))
prob := 1 / (1 + math.Pow(math.E, eta))
return prob, len(topHam), len(topSpam), nil
@ -502,7 +502,7 @@ func (f *Filter) ClassifyMessagePath(ctx context.Context, path string) (probabil
}
func (f *Filter) ClassifyMessageReader(ctx context.Context, mf io.ReaderAt, size int64) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
m, err := message.EnsurePart(f.log, false, mf, size)
m, err := message.EnsurePart(f.log.Logger, false, mf, size)
if err != nil && errors.Is(err, message.ErrBadContentType) {
// Invalid content-type header is a sure sign of spam.
//f.log.Infox("parsing content", err)
@ -568,7 +568,7 @@ func (f *Filter) Train(ctx context.Context, ham bool, words map[string]struct{})
}
func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error {
p, _ := message.EnsurePart(f.log, false, r, size)
p, _ := message.EnsurePart(f.log.Logger, false, r, size)
words, err := f.ParseMessage(p)
if err != nil {
return fmt.Errorf("parsing mail contents: %v", err)
@ -577,7 +577,7 @@ func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ha
}
func (f *Filter) UntrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error {
p, _ := message.EnsurePart(f.log, false, r, size)
p, _ := message.EnsurePart(f.log.Logger, false, r, size)
words, err := f.ParseMessage(p)
if err != nil {
return fmt.Errorf("parsing mail contents: %v", err)
@ -648,7 +648,7 @@ func (f *Filter) TrainDir(dir string, files []string, ham bool) (n, malformed ui
p := filepath.Join(dir, name)
valid, words, err := f.tokenizeMail(p)
if err != nil {
// f.log.Infox("tokenizing mail", err, mlog.Field("path", p))
// f.log.Infox("tokenizing mail", err, slog.Any("path", p))
malformed++
continue
}
@ -720,21 +720,20 @@ func (f *Filter) TrainDirs(hamDir, sentDir, spamDir string, hamFiles, sentFiles,
dbSize := f.fileSize(f.dbPath)
bloomSize := f.fileSize(f.bloomPath)
fields := []mlog.Pair{
mlog.Field("hams", hams),
mlog.Field("hamtime", tham),
mlog.Field("hammalformed", hamMalformed),
mlog.Field("sent", sent),
mlog.Field("senttime", tsent),
mlog.Field("sentmalformed", sentMalformed),
mlog.Field("spams", f.spams),
mlog.Field("spamtime", tspam),
mlog.Field("spammalformed", spamMalformed),
mlog.Field("dbsize", fmt.Sprintf("%.1fmb", float64(dbSize)/(1024*1024))),
mlog.Field("bloomsize", fmt.Sprintf("%.1fmb", float64(bloomSize)/(1024*1024))),
mlog.Field("bloom1ratio", fmt.Sprintf("%.4f", float64(f.bloom.Ones())/float64(len(f.bloom.Bytes())*8))),
}
xlog.Print("training done", fields...)
f.log.Print("training done",
slog.Any("hams", hams),
slog.Any("hamtime", tham),
slog.Any("hammalformed", hamMalformed),
slog.Any("sent", sent),
slog.Any("senttime", tsent),
slog.Any("sentmalformed", sentMalformed),
slog.Any("spams", f.spams),
slog.Any("spamtime", tspam),
slog.Any("spammalformed", spamMalformed),
slog.Any("dbsize", fmt.Sprintf("%.1fmb", float64(dbSize)/(1024*1024))),
slog.Any("bloomsize", fmt.Sprintf("%.1fmb", float64(bloomSize)/(1024*1024))),
slog.Any("bloom1ratio", fmt.Sprintf("%.4f", float64(f.bloom.Ones())/float64(len(f.bloom.Bytes())*8))),
)
return nil
}
@ -742,7 +741,7 @@ func (f *Filter) TrainDirs(hamDir, sentDir, spamDir string, hamFiles, sentFiles,
func (f *Filter) fileSize(p string) int {
fi, err := os.Stat(p)
if err != nil {
f.log.Infox("stat", err, mlog.Field("path", p))
f.log.Infox("stat", err, slog.Any("path", p))
return 0
}
return int(fi.Size())

2
junk/filter_test.go
View file

@ -32,7 +32,7 @@ func tlistdir(t *testing.T, name string) []string {
}
func TestFilter(t *testing.T) {
log := mlog.New("junk")
log := mlog.New("junk", nil)
params := Params{
Onegrams: true,
Twograms: true,

2
junk/parse.go
View file

@ -31,7 +31,7 @@ func (f *Filter) tokenizeMail(path string) (bool, map[string]struct{}, error) {
if err != nil {
return false, nil, err
}
p, _ := message.EnsurePart(f.log, false, mf, fi.Size())
p, _ := message.EnsurePart(f.log.Logger, false, mf, fi.Size())
words, err := f.ParseMessage(p)
return true, words, err
}

5
junk/parse_test.go
View file

@ -4,6 +4,8 @@ import (
"os"
"path/filepath"
"testing"
"github.com/mjl-/mox/mlog"
)
func FuzzParseMessage(f *testing.F) {
@ -24,7 +26,8 @@ func FuzzParseMessage(f *testing.F) {
os.Remove(dbPath)
os.Remove(bloomPath)
params := Params{Twograms: true}
jf, err := NewFilter(ctxbg, xlog, params, dbPath, bloomPath)
log := mlog.New("junk", nil)
jf, err := NewFilter(ctxbg, log, params, dbPath, bloomPath)
if err != nil {
f.Fatalf("new filter: %v", err)
}

30
localserve.go
View file

@ -20,6 +20,7 @@ import (
"time"
"golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slog"
"github.com/mjl-/sconf"
@ -75,18 +76,17 @@ during those commands instead of during "data".
c.Usage()
}
log := mlog.New("localserve")
log := c.log
mox.FilesImmediate = true
if initOnly {
if _, err := os.Stat(dir); err == nil {
log.Print("warning: directory for configuration files already exists, continuing")
}
log.Print("creating mox localserve config", mlog.Field("dir", dir))
log.Print("creating mox localserve config", slog.String("dir", dir))
err := writeLocalConfig(log, dir, ip)
if err != nil {
log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir))
log.Fatalx("creating mox localserve config", err, slog.String("dir", dir))
}
return
}
@ -96,12 +96,12 @@ during those commands instead of during "data".
if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
err := writeLocalConfig(log, dir, ip)
if err != nil {
log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir))
log.Fatalx("creating mox localserve config", err, slog.String("dir", dir))
}
} else if err != nil {
log.Fatalx("stat config dir", err, mlog.Field("dir", dir))
log.Fatalx("stat config dir", err, slog.String("dir", dir))
} else if err := localLoadConfig(log, dir); err != nil {
log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, mlog.Field("dir", dir))
log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, slog.String("dir", dir))
} else if ip != "" {
log.Fatal("can only use -ip when writing a new config file")
} else {
@ -112,7 +112,7 @@ during those commands instead of during "data".
mox.Conf.Log[""] = level
mlog.SetConfig(mox.Conf.Log)
} else if loglevel != "" && !ok {
log.Fatal("unknown loglevel", mlog.Field("loglevel", loglevel))
log.Fatal("unknown loglevel", slog.String("loglevel", loglevel))
}
// Initialize receivedid.
@ -201,7 +201,7 @@ during those commands instead of during "data".
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
sig := <-sigc
log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig))
log.Print("shutting down, waiting max 3s for existing connections", slog.Any("signal", sig))
shutdown(log)
if num, ok := sig.(syscall.Signal); ok {
os.Exit(int(num))
@ -210,7 +210,7 @@ during those commands instead of during "data".
}
}
func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
func writeLocalConfig(log mlog.Log, dir, ip string) (rerr error) {
defer func() {
x := recover()
if x != nil {
@ -220,7 +220,7 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
}
if rerr != nil {
err := os.RemoveAll(dir)
log.Check(err, "removing config directory", mlog.Field("dir", dir))
log.Check(err, "removing config directory", slog.String("dir", dir))
}
}()
@ -430,10 +430,10 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
xcheck(err, "loading config")
// Set password on account.
a, _, err := store.OpenEmail("mox@localhost")
a, _, err := store.OpenEmail(log, "mox@localhost")
xcheck(err, "opening account to set password")
password := "moxmoxmox"
err = a.SetPassword(password)
err = a.SetPassword(log, password)
xcheck(err, "setting password")
err = a.Close()
xcheck(err, "closing account")
@ -442,10 +442,10 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) {
return nil
}
func localLoadConfig(log *mlog.Log, dir string) error {
func localLoadConfig(log mlog.Log, dir string) error {
mox.ConfigStaticPath = filepath.Join(dir, "mox.conf")
mox.ConfigDynamicPath = filepath.Join(dir, "domains.conf")
errs := mox.LoadConfig(context.Background(), true, false)
errs := mox.LoadConfig(context.Background(), log, true, false)
if len(errs) > 1 {
log.Error("loading config generated config file: multiple errors")
for _, err := range errs {

111
main.go
View file

@ -32,6 +32,7 @@ import (
"golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"github.com/mjl-/adns"
@ -211,6 +212,8 @@ type cmd struct {
params string // Arguments to command. Multiple lines possible.
help string // Additional explanation. First line is synopsis, the rest is only printed for an explicit help/usage for that command.
args []string
log mlog.Log
}
func (c *cmd) Parse() []string {
@ -388,7 +391,7 @@ func mustLoadConfig() {
mox.Conf.Log[""] = level
mlog.SetConfig(mox.Conf.Log)
} else if loglevel != "" && !ok {
log.Fatal("unknown loglevel", mlog.Field("loglevel", loglevel))
log.Fatal("unknown loglevel", slog.String("loglevel", loglevel))
}
if pedantic {
moxvar.Pedantic = true
@ -413,6 +416,7 @@ func main() {
c := &cmd{
flag: flag.NewFlagSet("sendmail", flag.ExitOnError),
flagArgs: os.Args[1:],
log: mlog.New("sendmail", nil),
}
cmdSendmail(c)
return
@ -464,6 +468,7 @@ next:
}
c.flag = flag.NewFlagSet("mox "+strings.Join(c.words, " "), flag.ExitOnError)
c.flagArgs = args[len(c.words):]
c.log = mlog.New(strings.Join(c.words, ""), nil)
c.fn(&c)
return
}
@ -538,7 +543,7 @@ are printed.
mox.FilesImmediate = true
_, errs := mox.ParseConfig(context.Background(), mox.ConfigStaticPath, true, true, false)
_, errs := mox.ParseConfig(context.Background(), c.log, mox.ConfigStaticPath, true, true, false)
if len(errs) > 1 {
log.Printf("multiple errors:")
for _, err := range errs {
@ -1596,7 +1601,7 @@ connection.
}
resolver := dns.StrictResolver{Pkg: "danedial"}
conn, record, err := dane.Dial(context.Background(), resolver, "tcp", args[0], allowedUsages)
conn, record, err := dane.Dial(context.Background(), c.log.Logger, resolver, "tcp", args[0], allowedUsages)
xcheckf(err, "dial")
log.Printf("(connected, verified with %s)", record)
@ -1644,8 +1649,6 @@ sharing most of its code.
origNextHop, err := dns.ParseDomain(args[0])
xcheckf(err, "parse domain")
clog := mlog.New("danedialmx")
ctxbg := context.Background()
resolver := dns.StrictResolver{}
@ -1655,7 +1658,7 @@ sharing most of its code.
var hosts []dns.IPDomain
if len(args) == 1 {
var permanent bool
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err = smtpclient.GatherDestinations(ctxbg, clog, resolver, dns.IPDomain{Domain: origNextHop})
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err = smtpclient.GatherDestinations(ctxbg, c.log.Logger, resolver, dns.IPDomain{Domain: origNextHop})
status := "temporary"
if permanent {
status = "permanent"
@ -1706,7 +1709,7 @@ sharing most of its code.
log.Printf("attempting to connect to %s", host)
authentic, expandedAuthentic, expandedHost, ips, _, err := smtpclient.GatherIPs(ctxbg, clog, resolver, host, dialedIPs)
authentic, expandedAuthentic, expandedHost, ips, _, err := smtpclient.GatherIPs(ctxbg, c.log.Logger, resolver, host, dialedIPs)
if err != nil {
log.Printf("resolving ips for %s: %v, skipping", host, err)
continue
@ -1724,7 +1727,7 @@ sharing most of its code.
}
log.Printf("host %s resolved to ips %s, looking up tlsa records", host, ips)
daneRequired, daneRecords, tlsaBaseDomain, err := smtpclient.GatherTLSA(ctxbg, clog, resolver, host.Domain, expandedAuthentic, expandedHost)
daneRequired, daneRecords, tlsaBaseDomain, err := smtpclient.GatherTLSA(ctxbg, c.log.Logger, resolver, host.Domain, expandedAuthentic, expandedHost)
if err != nil {
log.Printf("looking up tlsa records: %s, skipping", err)
continue
@ -1753,7 +1756,7 @@ sharing most of its code.
log.Printf("gathered valid tls certificate names for potential verification with dane-ta: %s", strings.Join(l, ", "))
dialer := &net.Dialer{Timeout: 5 * time.Second}
conn, _, err := smtpclient.Dial(ctxbg, clog, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs)
conn, _, err := smtpclient.Dial(ctxbg, c.log.Logger, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs)
if err != nil {
log.Printf("dial %s: %v, skipping", expandedHost, err)
continue
@ -1768,7 +1771,7 @@ sharing most of its code.
RootCAs: mox.Conf.Static.TLS.CertPool,
}
tlsPKIX := false
sc, err := smtpclient.New(ctxbg, clog, conn, tlsMode, tlsPKIX, ehloDomain, tlsHostnames[0], opts)
sc, err := smtpclient.New(ctxbg, c.log.Logger, conn, tlsMode, tlsPKIX, ehloDomain, tlsHostnames[0], opts)
if err != nil {
log.Printf("setting up smtp session: %v, skipping", err)
conn.Close()
@ -2175,7 +2178,7 @@ that was passed.
msgf, err := os.Open(args[0])
xcheckf(err, "open message")
results, err := dkim.Verify(context.Background(), dns.StrictResolver{}, false, dkim.DefaultPolicy, msgf, true)
results, err := dkim.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, false, dkim.DefaultPolicy, msgf, true)
xcheckf(err, "dkim verify")
for _, result := range results {
@ -2214,13 +2217,11 @@ headers prepended.
c.Usage()
}
clog := mlog.New("dkimsign")
msgf, err := os.Open(args[0])
xcheckf(err, "open message")
defer msgf.Close()
p, err := message.Parse(clog, true, msgf)
p, err := message.Parse(c.log.Logger, true, msgf)
xcheckf(err, "parsing message")
if len(p.Envelope.From) != 1 {
@ -2237,7 +2238,7 @@ headers prepended.
log.Fatalf("domain %s not configured", dom)
}
headers, err := dkim.Sign(context.Background(), localpart, dom, domConf.DKIM, false, msgf)
headers, err := dkim.Sign(context.Background(), c.log.Logger, localpart, dom, domConf.DKIM, false, msgf)
xcheckf(err, "signing message with dkim")
if headers == "" {
log.Fatalf("no DKIM configured for domain %s", dom)
@ -2259,7 +2260,7 @@ func cmdDKIMLookup(c *cmd) {
selector := xparseDomain(args[0], "selector")
domain := xparseDomain(args[1], "domain")
status, record, txt, authentic, err := dkim.Lookup(context.Background(), dns.StrictResolver{}, selector, domain)
status, record, txt, authentic, err := dkim.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, selector, domain)
if err != nil {
fmt.Printf("error: %s\n", err)
}
@ -2299,7 +2300,7 @@ func cmdDMARCLookup(c *cmd) {
}
fromdomain := xparseDomain(args[0], "domain")
_, domain, _, txt, authentic, err := dmarc.Lookup(context.Background(), dns.StrictResolver{}, fromdomain)
_, domain, _, txt, authentic, err := dmarc.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, fromdomain)
xcheckf(err, "dmarc lookup domain %s", fromdomain)
fmt.Printf("dmarc record at domain %s: %s\n", domain, txt)
fmt.Printf("(%s)\n", dnssecStatus(authentic))
@ -2359,7 +2360,7 @@ can be found in message headers.
if heloDomain != nil {
spfArgs.HelloDomain = dns.IPDomain{Domain: *heloDomain}
}
rspf, spfDomain, expl, authentic, err := spf.Verify(context.Background(), dns.StrictResolver{}, spfArgs)
rspf, spfDomain, expl, authentic, err := spf.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, spfArgs)
if err != nil {
log.Printf("spf verify: %v (explanation: %q, authentic %v)", err, expl, authentic)
} else {
@ -2377,17 +2378,17 @@ can be found in message headers.
data, err := io.ReadAll(os.Stdin)
xcheckf(err, "read message")
dmarcFrom, _, _, err := message.From(mlog.New("dmarcverify"), false, bytes.NewReader(data))
dmarcFrom, _, _, err := message.From(c.log.Logger, false, bytes.NewReader(data))
xcheckf(err, "extract dmarc from message")
const ignoreTestMode = false
dkimResults, err := dkim.Verify(context.Background(), dns.StrictResolver{}, true, func(*dkim.Sig) error { return nil }, bytes.NewReader(data), ignoreTestMode)
dkimResults, err := dkim.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, true, func(*dkim.Sig) error { return nil }, bytes.NewReader(data), ignoreTestMode)
xcheckf(err, "dkim verify")
for _, r := range dkimResults {
fmt.Printf("dkim result: %q (err %v)\n", r.Status, r.Err)
}
_, result := dmarc.Verify(context.Background(), dns.StrictResolver{}, dmarcFrom.Domain, dkimResults, spfStatus, spfIdentity, false)
_, result := dmarc.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, dmarcFrom.Domain, dkimResults, spfStatus, spfIdentity, false)
xcheckf(result.Err, "dmarc verify")
fmt.Printf("dmarc from: %s\ndmarc status: %q\ndmarc reject: %v\ncmarc record: %s\n", dmarcFrom, result.Status, result.Reject, result.Record)
}
@ -2408,7 +2409,7 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at
}
dom := xparseDomain(args[0], "domain")
_, domain, record, txt, authentic, err := dmarc.Lookup(context.Background(), dns.StrictResolver{}, dom)
_, domain, record, txt, authentic, err := dmarc.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, dom)
xcheckf(err, "dmarc lookup domain %s", dom)
fmt.Printf("dmarc record at domain %s: %q\n", domain, txt)
fmt.Printf("(%s)\n", dnssecStatus(authentic))
@ -2439,12 +2440,12 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at
return
}
if publicsuffix.Lookup(context.Background(), dom) == publicsuffix.Lookup(context.Background(), destdom) {
if publicsuffix.Lookup(context.Background(), c.log.Logger, dom) == publicsuffix.Lookup(context.Background(), c.log.Logger, destdom) {
printResult("pass (same organizational domain)")
return
}
accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom)
accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), c.log.Logger, dns.StrictResolver{}, domain, destdom)
var txtstr string
txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII)
if len(txts) == 0 {
@ -2486,12 +2487,10 @@ understand email deliverability problems.
c.Usage()
}
clog := mlog.New("dmarcparsereportmsg")
for _, arg := range args {
f, err := os.Open(arg)
xcheckf(err, "open %q", arg)
feedback, err := dmarcrpt.ParseMessageReport(clog, f)
feedback, err := dmarcrpt.ParseMessageReport(c.log.Logger, f)
xcheckf(err, "parse report in %q", arg)
meta := feedback.ReportMetadata
fmt.Printf("Report: period %s-%s, organisation %q, reportID %q, %s\n", time.Unix(meta.DateRange.Begin, 0).UTC().String(), time.Unix(meta.DateRange.End, 0).UTC().String(), meta.OrgName, meta.ReportID, meta.Email)
@ -2540,11 +2539,9 @@ func cmdDMARCDBAddReport(c *cmd) {
mustLoadConfig()
clog := mlog.New("dmarcdbaddreport")
fromdomain := xparseDomain(args[0], "domain")
fmt.Fprintln(os.Stderr, "reading report message from stdin")
report, err := dmarcrpt.ParseMessageReport(clog, os.Stdin)
report, err := dmarcrpt.ParseMessageReport(c.log.Logger, os.Stdin)
xcheckf(err, "parse message")
err = dmarcdb.AddReport(context.Background(), report, fromdomain)
xcheckf(err, "add dmarc report")
@ -2565,7 +2562,7 @@ successfully used TLS, and how what kind of errors occurred otherwise.
}
d := xparseDomain(args[0], "domain")
_, txt, err := tlsrpt.Lookup(context.Background(), dns.StrictResolver{}, d)
_, txt, err := tlsrpt.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, d)
xcheckf(err, "tlsrpt lookup for %s", d)
fmt.Println(txt)
}
@ -2581,12 +2578,10 @@ The report is printed in formatted JSON.
c.Usage()
}
clog := mlog.New("tlsrptparsereportmsg")
for _, arg := range args {
f, err := os.Open(arg)
xcheckf(err, "open %q", arg)
report, err := tlsrpt.ParseMessage(clog, f)
report, err := tlsrpt.ParseMessage(c.log.Logger, f)
xcheckf(err, "parse report in %q", arg)
// todo future: only print the highlights?
enc := json.NewEncoder(os.Stdout)
@ -2622,7 +2617,7 @@ printed.
LocalIP: net.ParseIP("127.0.0.1"),
LocalHostname: dns.Domain{ASCII: "localhost"},
}
r, _, explanation, authentic, err := spf.Verify(context.Background(), dns.StrictResolver{}, spfargs)
r, _, explanation, authentic, err := spf.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, spfargs)
if err != nil {
fmt.Printf("error: %s\n", err)
}
@ -2656,7 +2651,7 @@ func cmdSPFLookup(c *cmd) {
}
domain := xparseDomain(args[0], "domain")
_, txt, _, authentic, err := spf.Lookup(context.Background(), dns.StrictResolver{}, domain)
_, txt, _, authentic, err := spf.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, domain)
xcheckf(err, "spf lookup for %s", domain)
fmt.Println(txt)
fmt.Printf("(%s)\n", dnssecStatus(authentic))
@ -2680,7 +2675,7 @@ should be used, and how long the policy can be cached.
domain := xparseDomain(args[0], "domain")
record, policy, _, err := mtasts.Get(context.Background(), dns.StrictResolver{}, domain)
record, policy, _, err := mtasts.Get(context.Background(), c.log.Logger, dns.StrictResolver{}, domain)
if err != nil {
fmt.Printf("error: %s\n", err)
}
@ -2729,13 +2724,11 @@ func cmdTLSRPTDBAddReport(c *cmd) {
mustLoadConfig()
clog := mlog.New("tlsrptdbaddreport")
// First read message, to get the From-header. Then parse it as TLSRPT.
fmt.Fprintln(os.Stderr, "reading report message from stdin")
buf, err := io.ReadAll(os.Stdin)
xcheckf(err, "reading message")
part, err := message.Parse(clog, true, bytes.NewReader(buf))
part, err := message.Parse(c.log.Logger, true, bytes.NewReader(buf))
xcheckf(err, "parsing message")
if part.Envelope == nil || len(part.Envelope.From) != 1 {
log.Fatalf("message must have one From-header")
@ -2743,11 +2736,11 @@ func cmdTLSRPTDBAddReport(c *cmd) {
from := part.Envelope.From[0]
domain := xparseDomain(from.Host, "domain")
report, err := tlsrpt.ParseMessage(clog, bytes.NewReader(buf))
report, err := tlsrpt.ParseMessage(c.log.Logger, bytes.NewReader(buf))
xcheckf(err, "parsing tls report in message")
mailfrom := from.User + "@" + from.Host // todo future: should escape and such
err = tlsrptdb.AddReport(context.Background(), domain, mailfrom, hostReport, report)
err = tlsrptdb.AddReport(context.Background(), c.log, domain, mailfrom, hostReport, report)
xcheckf(err, "add tls report to database")
}
@ -2766,7 +2759,7 @@ URL with more information.
zone := xparseDomain(args[0], "zone")
ip := xparseIP(args[1], "ip")
status, explanation, err := dnsbl.Lookup(context.Background(), dns.StrictResolver{}, zone, ip)
status, explanation, err := dnsbl.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, zone, ip)
fmt.Printf("status: %s\n", status)
if status == dnsbl.StatusFail {
fmt.Printf("explanation: %q\n", explanation)
@ -2789,7 +2782,7 @@ The health of a DNS blocklist can be checked by querying for 127.0.0.1 and
}
zone := xparseDomain(args[0], "zone")
err := dnsbl.CheckHealth(context.Background(), dns.StrictResolver{}, zone)
err := dnsbl.CheckHealth(context.Background(), c.log.Logger, dns.StrictResolver{}, zone)
xcheckf(err, "unhealthy")
fmt.Println("healthy")
}
@ -2814,12 +2807,12 @@ printed.
fmt.Printf("last known version: %s\n", lastknown)
fmt.Printf("current version: %s\n", current)
}
latest, _, err := updates.Lookup(context.Background(), dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain})
latest, _, err := updates.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain})
xcheckf(err, "lookup of latest version")
fmt.Printf("latest version: %s\n", latest)
if latest.After(current) {
changelog, err := updates.FetchChangelog(context.Background(), changelogURL, current, changelogPubKey)
changelog, err := updates.FetchChangelog(context.Background(), c.log.Logger, changelogURL, current, changelogPubKey)
xcheckf(err, "fetching changelog")
if len(changelog.Changes) == 0 {
log.Printf("no changes in changelog")
@ -2884,7 +2877,7 @@ open, or is not running.
}
mustLoadConfig()
a, err := store.OpenAccount(args[0])
a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account")
defer func() {
if err := a.Close(); err != nil {
@ -2942,7 +2935,7 @@ open, or is not running.
}
mustLoadConfig()
a, err := store.OpenAccount(args[0])
a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account")
defer func() {
if err := a.Close(); err != nil {
@ -3036,7 +3029,7 @@ open, or is not running.
}
mustLoadConfig()
a, err := store.OpenAccount(args[0])
a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account")
defer func() {
if err := a.Close(); err != nil {
@ -3156,10 +3149,8 @@ func cmdEnsureParsed(c *cmd) {
c.Usage()
}
clog := mlog.New("ensureparsed")
mustLoadConfig()
a, err := store.OpenAccount(args[0])
a, err := store.OpenAccount(c.log, args[0])
xcheckf(err, "open account")
defer func() {
if err := a.Close(); err != nil {
@ -3180,7 +3171,7 @@ func cmdEnsureParsed(c *cmd) {
}
for _, m := range l {
mr := a.MessageReader(m)
p, err := message.EnsurePart(clog, false, mr, m.Size)
p, err := message.EnsurePart(c.log.Logger, false, mr, m.Size)
if err != nil {
log.Printf("parsing message %d: %v (continuing)", m.ID, err)
}
@ -3233,15 +3224,13 @@ func cmdMessageParse(c *cmd) {
c.Usage()
}
clog := mlog.New("messageparse")
f, err := os.Open(args[0])
xcheckf(err, "open")
defer f.Close()
part, err := message.Parse(clog, false, f)
part, err := message.Parse(c.log.Logger, false, f)
xcheckf(err, "parsing message")
err = part.Walk(clog, nil)
err = part.Walk(c.log.Logger, nil)
xcheckf(err, "parsing nested parts")
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", "\t")
@ -3262,15 +3251,13 @@ Opens database files directly, not going through a running mox instance.
c.Usage()
}
clog := mlog.New("openaccounts")
dataDir := filepath.Clean(args[0])
for _, accName := range args[1:] {
accDir := filepath.Join(dataDir, "accounts", accName)
log.Printf("opening account %s...", accDir)
a, err := store.OpenAccountDB(accDir, accName)
a, err := store.OpenAccountDB(c.log, accDir, accName)
xcheckf(err, "open account %s", accName)
err = a.ThreadingWait(clog)
err = a.ThreadingWait(c.log)
xcheckf(err, "wait for threading upgrade to complete for %s", accName)
err = a.Close()
xcheckf(err, "close account %s", accName)
@ -3363,7 +3350,7 @@ Opens database files directly, not going through a running mox instance.
for _, accName := range args[1:] {
accDir := filepath.Join(dataDir, "accounts", accName)
log.Printf("opening account %s...", accDir)
a, err := store.OpenAccountDB(accDir, accName)
a, err := store.OpenAccountDB(c.log, accDir, accName)
xcheckf(err, "open account %s", accName)
prepareMessages := func(in, out chan moxio.Work[store.Message, threadPrep]) {

8
message/from.go
View file

@ -5,6 +5,8 @@ import (
"io"
"net/textproto"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp"
@ -17,12 +19,14 @@ import (
// From headers may be present. From returns an error if there is not exactly
// one address. This address can be used for evaluating a DMARC policy against
// SPF and DKIM results.
func From(log *mlog.Log, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) {
func From(elog *slog.Logger, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) {
log := mlog.New("message", elog)
// ../rfc/7489:1243
// todo: only allow utf8 if enabled in session/message?
p, err := Parse(log, strict, r)
p, err := Parse(log.Logger, strict, r)
if err != nil {
// todo: should we continue with p, perhaps headers can be parsed?
return raddr, nil, nil, fmt.Errorf("parsing message: %v", err)

39
message/part.go
View file

@ -21,6 +21,7 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"golang.org/x/text/encoding/ianaindex"
"github.com/mjl-/mox/mlog"
@ -111,7 +112,8 @@ type Address struct {
//
// If strict is set, fewer attempts are made to continue parsing when errors are
// encountered, such as with invalid content-type headers or bare carriage returns.
func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) {
func Parse(elog *slog.Logger, strict bool, r io.ReaderAt) (Part, error) {
log := mlog.New("message", elog)
return newPart(log, strict, r, 0, nil)
}
@ -122,10 +124,11 @@ func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) {
//
// If strict is set, fewer attempts are made to continue parsing when errors are
// encountered, such as with invalid content-type headers or bare carriage returns.
func EnsurePart(log *mlog.Log, strict bool, r io.ReaderAt, size int64) (Part, error) {
p, err := Parse(log, strict, r)
func EnsurePart(elog *slog.Logger, strict bool, r io.ReaderAt, size int64) (Part, error) {
log := mlog.New("message", elog)
p, err := Parse(log.Logger, strict, r)
if err == nil {
err = p.Walk(log, nil)
err = p.Walk(log.Logger, nil)
}
if err != nil {
np, err2 := fallbackPart(p, r, size)
@ -185,7 +188,9 @@ func (p *Part) SetMessageReaderAt() error {
}
// Walk through message, decoding along the way, and collecting mime part offsets and sizes, and line counts.
func (p *Part) Walk(log *mlog.Log, parent *Part) error {
func (p *Part) Walk(elog *slog.Logger, parent *Part) error {
log := mlog.New("message", elog)
if len(p.bound) == 0 {
if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {
// todo: don't read whole submessage in memory...
@ -194,11 +199,11 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error {
return err
}
br := bytes.NewReader(buf)
mp, err := Parse(log, p.strict, br)
mp, err := Parse(log.Logger, p.strict, br)
if err != nil {
return fmt.Errorf("parsing embedded message: %w", err)
}
if err := mp.Walk(log, nil); err != nil {
if err := mp.Walk(log.Logger, nil); err != nil {
// If this is a DSN and we are not in pedantic mode, accept unexpected end of
// message. This is quite common because MTA's sometimes just truncate the original
// message in a place that makes the message invalid.
@ -220,14 +225,14 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error {
}
for {
pp, err := p.ParseNextPart(log)
pp, err := p.ParseNextPart(log.Logger)
if err == io.EOF {
return nil
}
if err != nil {
return err
}
if err := pp.Walk(log, p); err != nil {
if err := pp.Walk(log.Logger, p); err != nil {
return err
}
}
@ -241,7 +246,7 @@ func (p *Part) String() string {
// newPart parses a new part, which can be the top-level message.
// offset is the bound offset for parts, and the start of message for top-level messages. parent indicates if this is a top-level message or sub-part.
// If an error occurs, p's exported values can still be relevant. EnsurePart uses these values.
func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) {
func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) {
if r == nil {
panic("nil reader")
}
@ -325,14 +330,14 @@ func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Pa
p.MediaType = "APPLICATION"
p.MediaSubType = "OCTET-STREAM"
}
log.Debugx("malformed content-type, attempting to recover and continuing", err, mlog.Field("contenttype", p.header.Get("Content-Type")), mlog.Field("mediatype", p.MediaType), mlog.Field("mediasubtype", p.MediaSubType))
log.Debugx("malformed content-type, attempting to recover and continuing", err, slog.String("contenttype", p.header.Get("Content-Type")), slog.String("mediatype", p.MediaType), slog.String("mediasubtype", p.MediaSubType))
} else if mt != "" {
t := strings.SplitN(strings.ToUpper(mt), "/", 2)
if len(t) != 2 {
if moxvar.Pedantic || strict {
return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct)
}
log.Debug("malformed media-type, ignoring and continuing", mlog.Field("type", mt))
log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt))
p.MediaType = "APPLICATION"
p.MediaSubType = "OCTET-STREAM"
} else {
@ -444,7 +449,7 @@ var wordDecoder = mime.WordDecoder{
},
}
func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) {
func parseEnvelope(log mlog.Log, h mail.Header) (*Envelope, error) {
date, _ := h.Date()
// We currently marshal this field to JSON. But JSON cannot represent all
@ -478,7 +483,7 @@ func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) {
return env, nil
}
func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
func parseAddressList(log mlog.Log, h mail.Header, k string) []Address {
// todo: possibly work around ios mail generating incorrect q-encoded "phrases" with unencoded double quotes? ../rfc/2047:382
l, err := h.AddressList(k)
if err != nil {
@ -490,7 +495,7 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
var user, host string
addr, err := smtp.ParseAddress(a.Address)
if err != nil {
log.Infox("parsing address (continuing)", err, mlog.Field("address", a.Address))
log.Infox("parsing address (continuing)", err, slog.Any("address", a.Address))
} else {
user = addr.Localpart.String()
host = addr.Domain.ASCII
@ -503,7 +508,9 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address {
// ParseNextPart parses the next (sub)part of this multipart message.
// ParseNextPart returns io.EOF and a nil part when there are no more parts.
// Only used for initial parsing of message. Once parsed, use p.Parts.
func (p *Part) ParseNextPart(log *mlog.Log) (*Part, error) {
func (p *Part) ParseNextPart(elog *slog.Logger) (*Part, error) {
log := mlog.New("message", elog)
if len(p.bound) == 0 {
return nil, errNotMultipart
}

74
message/part_test.go
View file

@ -15,7 +15,7 @@ import (
"github.com/mjl-/mox/moxvar"
)
var xlog = mlog.New("message")
var pkglog = mlog.New("message", nil)
func tcheck(t *testing.T, err error, msg string) {
t.Helper()
@ -40,7 +40,7 @@ func tfail(t *testing.T, err, expErr error) {
func TestEmptyHeader(t *testing.T) {
s := "\r\nx"
p, err := EnsurePart(xlog, true, strings.NewReader(s), int64(len(s)))
p, err := EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parse empty headers")
buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -56,7 +56,7 @@ func TestBadContentType(t *testing.T) {
// Pedantic is like strict.
moxvar.Pedantic = true
s := "content-type: text/html;;\r\n\r\ntest"
p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType)
buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -67,7 +67,7 @@ func TestBadContentType(t *testing.T) {
// Strict
s = "content-type: text/html;;\r\n\r\ntest"
p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType)
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -77,7 +77,7 @@ func TestBadContentType(t *testing.T) {
// Non-strict but unrecoverable content-type.
s = "content-type: not a content type;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -87,7 +87,7 @@ func TestBadContentType(t *testing.T) {
// We try to use only the content-type, typically better than application/octet-stream.
s = "content-type: text/html;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -97,7 +97,7 @@ func TestBadContentType(t *testing.T) {
// Not recovering multipart, we won't have a boundary.
s = "content-type: multipart/mixed;;\r\n\r\ntest"
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parsing message with bad but recoverable content-type")
buf, err = io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -112,20 +112,20 @@ func TestBareCR(t *testing.T) {
// Pedantic is like strict.
moxvar.Pedantic = true
p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader())
tfail(t, err, errBareCR)
moxvar.Pedantic = false
// Strict.
p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader())
tcheck(t, err, "read fallback part without error")
// Non-strict allows bare cr.
p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s)))
p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tcheck(t, err, "parse")
buf, err := io.ReadAll(p.Reader())
tcheck(t, err, "read")
@ -141,7 +141,7 @@ aGkK
func TestBasic(t *testing.T) {
r := strings.NewReader(basicMsg)
p, err := Parse(xlog, true, r)
p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
buf, err := io.ReadAll(p.RawReader())
@ -176,7 +176,7 @@ Hello Joe, do you think we can meet at 3:30 tomorrow?
func TestBasic2(t *testing.T) {
r := strings.NewReader(basicMsg2)
p, err := Parse(xlog, true, r)
p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
buf, err := io.ReadAll(p.RawReader())
@ -196,9 +196,9 @@ func TestBasic2(t *testing.T) {
}
r = strings.NewReader(basicMsg2)
p, err = Parse(xlog, true, r)
p, err = Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
err = p.Walk(xlog, nil)
err = p.Walk(pkglog.Logger, nil)
tcheck(t, err, "walk")
if p.RawLineCount != 2 {
t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount)
@ -237,25 +237,25 @@ This is the epilogue. It is also to be ignored.
func TestMime(t *testing.T) {
// from ../rfc/2046:1148
r := strings.NewReader(mimeMsg)
p, err := Parse(xlog, true, r)
p, err := Parse(pkglog.Logger, true, r)
tcheck(t, err, "new reader")
if len(p.bound) == 0 {
t.Fatalf("got no bound, expected bound for mime message")
}
pp, err := p.ParseNextPart(xlog)
pp, err := p.ParseNextPart(pkglog.Logger)
tcheck(t, err, "next part")
buf, err := io.ReadAll(pp.Reader())
tcheck(t, err, "read all")
tcompare(t, string(buf), "This is implicitly typed plain US-ASCII text.\r\nIt does NOT end with a linebreak.")
pp, err = p.ParseNextPart(xlog)
pp, err = p.ParseNextPart(pkglog.Logger)
tcheck(t, err, "next part")
buf, err = io.ReadAll(pp.Reader())
tcheck(t, err, "read all")
tcompare(t, string(buf), "This is explicitly typed plain US-ASCII text.\r\nIt DOES end with a linebreak.\r\n")
_, err = p.ParseNextPart(xlog)
_, err = p.ParseNextPart(pkglog.Logger)
tcompare(t, err, io.EOF)
if len(p.Parts) != 2 {
@ -274,17 +274,17 @@ func TestLongLine(t *testing.T) {
for i := range line {
line[i] = 'a'
}
_, err := Parse(xlog, true, bytes.NewReader(line))
_, err := Parse(pkglog.Logger, true, bytes.NewReader(line))
tfail(t, err, errLineTooLong)
}
func TestBareCrLf(t *testing.T) {
parse := func(strict bool, s string) error {
p, err := Parse(xlog, strict, strings.NewReader(s))
p, err := Parse(pkglog.Logger, strict, strings.NewReader(s))
if err != nil {
return err
}
return p.Walk(xlog, nil)
return p.Walk(pkglog.Logger, nil)
}
err := parse(false, "subject: test\ntest\r\n")
tfail(t, err, errBareLF)
@ -316,25 +316,25 @@ func TestMissingClosingBoundary(t *testing.T) {
test
`, "\n", "\r\n")
msg, err := Parse(xlog, false, strings.NewReader(message))
msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tcheck(t, err, "new reader")
err = walkmsg(&msg)
tfail(t, err, errMissingClosingBoundary)
msg, _ = Parse(xlog, false, strings.NewReader(message))
err = msg.Walk(xlog, nil)
msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
err = msg.Walk(pkglog.Logger, nil)
tfail(t, err, errMissingClosingBoundary)
}
func TestHeaderEOF(t *testing.T) {
message := "header: test"
_, err := Parse(xlog, false, strings.NewReader(message))
_, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tfail(t, err, errUnexpectedEOF)
}
func TestBodyEOF(t *testing.T) {
message := "header: test\r\n\r\ntest"
msg, err := Parse(xlog, true, strings.NewReader(message))
msg, err := Parse(pkglog.Logger, true, strings.NewReader(message))
tcheck(t, err, "new reader")
buf, err := io.ReadAll(msg.Reader())
tcheck(t, err, "read body")
@ -365,7 +365,7 @@ test
`, "\n", "\r\n")
msg, err := Parse(xlog, false, strings.NewReader(message))
msg, err := Parse(pkglog.Logger, false, strings.NewReader(message))
tcheck(t, err, "new reader")
enforceSequential = true
defer func() {
@ -374,8 +374,8 @@ test
err = walkmsg(&msg)
tcheck(t, err, "walkmsg")
msg, _ = Parse(xlog, false, strings.NewReader(message))
err = msg.Walk(xlog, nil)
msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message))
err = msg.Walk(pkglog.Logger, nil)
tcheck(t, err, "msg.Walk")
}
@ -452,7 +452,7 @@ Content-Transfer-Encoding: Quoted-printable
--unique-boundary-1--
`, "\n", "\r\n")
msg, err := Parse(xlog, true, strings.NewReader(nestedMessage))
msg, err := Parse(pkglog.Logger, true, strings.NewReader(nestedMessage))
tcheck(t, err, "new reader")
enforceSequential = true
defer func() {
@ -477,8 +477,8 @@ Content-Transfer-Encoding: Quoted-printable
t.Fatalf("got %q, expected %q", buf, exp)
}
msg, _ = Parse(xlog, false, strings.NewReader(nestedMessage))
err = msg.Walk(xlog, nil)
msg, _ = Parse(pkglog.Logger, false, strings.NewReader(nestedMessage))
err = msg.Walk(pkglog.Logger, nil)
tcheck(t, err, "msg.Walk")
}
@ -518,7 +518,7 @@ func walk(path string) error {
return err
}
defer r.Close()
msg, err := Parse(xlog, false, r)
msg, err := Parse(pkglog.Logger, false, r)
if err != nil {
return err
}
@ -538,7 +538,7 @@ func walkmsg(msg *Part) error {
}
if msg.MediaType == "MESSAGE" && (msg.MediaSubType == "RFC822" || msg.MediaSubType == "GLOBAL") {
mp, err := Parse(xlog, false, bytes.NewReader(buf))
mp, err := Parse(pkglog.Logger, false, bytes.NewReader(buf))
if err != nil {
return err
}
@ -566,7 +566,7 @@ func walkmsg(msg *Part) error {
}
for {
pp, err := msg.ParseNextPart(xlog)
pp, err := msg.ParseNextPart(pkglog.Logger)
if err == io.EOF {
return nil
}
@ -585,7 +585,7 @@ func TestEmbedded(t *testing.T) {
tcheck(t, err, "open")
fi, err := f.Stat()
tcheck(t, err, "stat")
_, err = EnsurePart(xlog, false, f, fi.Size())
_, err = EnsurePart(pkglog.Logger, false, f, fi.Size())
tcheck(t, err, "parse")
}
@ -594,6 +594,6 @@ func TestEmbedded2(t *testing.T) {
tcheck(t, err, "readfile")
buf = bytes.ReplaceAll(buf, []byte("\n"), []byte("\r\n"))
_, err = EnsurePart(xlog, false, bytes.NewReader(buf), int64(len(buf)))
_, err = EnsurePart(pkglog.Logger, false, bytes.NewReader(buf), int64(len(buf)))
tfail(t, err, nil)
}

2
message/referencedids_test.go
View file

@ -9,7 +9,7 @@ func TestReferencedIDs(t *testing.T) {
check := func(msg string, expRefs []string) {
t.Helper()
p, err := Parse(xlog, true, strings.NewReader(msg))
p, err := Parse(pkglog.Logger, true, strings.NewReader(msg))
tcheck(t, err, "parsing message")
h, err := p.Header()

10
metrics/http.go
View file

@ -8,14 +8,14 @@ import (
"os"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/mlog"
)
var xlog = mlog.New("metrics")
var (
metricHTTPClient = promauto.NewHistogramVec(
prometheus.HistogramOpts{
@ -34,8 +34,8 @@ var (
// HTTPClientObserve tracks the result of an HTTP transaction in a metric, and
// logs the result.
func HTTPClientObserve(ctx context.Context, pkg, method string, statusCode int, err error, start time.Time) {
log := xlog.WithContext(ctx)
func HTTPClientObserve(ctx context.Context, log mlog.Log, pkg, method string, statusCode int, err error, start time.Time) {
log = log.WithPkg("metrics")
var result string
switch {
case err == nil:
@ -57,5 +57,5 @@ func HTTPClientObserve(ctx context.Context, pkg, method string, statusCode int,
result = "error"
}
metricHTTPClient.WithLabelValues(pkg, method, result, fmt.Sprintf("%d", statusCode)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("httpclient result", err, mlog.Field("pkg", pkg), mlog.Field("method", method), mlog.Field("code", statusCode), mlog.Field("duration", time.Since(start)))
log.Debugx("httpclient result", err, slog.String("pkg", pkg), slog.String("method", method), slog.Int("code", statusCode), slog.Duration("duration", time.Since(start)))
}

633
mlog/log.go
View file

@ -1,30 +1,21 @@
// Package mlog provides logging with log levels and fields.
// Package mlog providers helpers on top of slog.Logger.
//
// Each log level has a function to log with and without error.
// Each such function takes a varargs list of fields (key value pairs) to log.
// Variable data should be in fields. Logging strings themselves should be
// constant, for easier log processing (e.g. building metrics based on log
// messages).
// Packages of mox that are fit or use by external code take an *slog.Logger as
// parameter for logging. Internally, and packages not intended for reuse,
// logging is done with mlog.Log. It providers convenience functions for:
// logging error values, tracing (protocol messages), uncoditional printing
// optionally exiting.
//
// The log levels can be configured per originating package, e.g. smtpclient,
// imapserver. The configuration is application-global, so each Log instance
// uses the same log levels.
//
// Print* should be used for lines that always should be printed, regardless of
// configured log levels. Useful for startup logging and subcommands.
//
// Fatal* stops the program. Its log text is always printed.
// An mlog provides a handler for an mlog.Log for formatting log lines. Lines are
// logged as "logfmt" lines for "mox serve". For command-line tools, the lines are
// printed with colon-separated level, message and error, followed by
// semicolon-separated attributes.
package mlog
// todo: log with source=path:linenumber? and/or stacktrace (perhaps optional)
// todo: should we turn errors logged with an context.Canceled from a level error into level info?
// todo: rethink format. perhaps simply using %#v is more useful for many types?
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"os"
@ -32,91 +23,101 @@ import (
"strconv"
"strings"
"sync/atomic"
"time"
"golang.org/x/exp/slog"
)
var noctx = context.Background()
// Logfmt enabled output in logfmt, instead of output more suitable for
// command-line tools. Must be set early in a program lifecycle.
var Logfmt bool
type Level int
func (l Level) String() string {
return LevelStrings[l]
}
var LevelStrings = map[Level]string{
LevelPrint: "print",
LevelFatal: "fatal",
LevelError: "error",
LevelInfo: "info",
LevelDebug: "debug",
LevelTrace: "trace",
LevelTraceauth: "traceauth",
LevelTracedata: "tracedata",
}
var Levels = map[string]Level{
"print": LevelPrint,
"fatal": LevelFatal,
"error": LevelError,
"info": LevelInfo,
"debug": LevelDebug,
"trace": LevelTrace,
"traceauth": LevelTraceauth,
"tracedata": LevelTracedata,
}
const (
LevelPrint Level = 0 // Printed regardless of configured log level.
LevelFatal Level = 1 // Printed regardless of configured log level.
LevelError Level = 2
LevelInfo Level = 3
LevelDebug Level = 4
LevelTrace Level = 5
LevelTraceauth Level = 6
LevelTracedata Level = 7
)
// LogStringer is used when formatting field values during logging. If a value
// implements it, LogString is called for the value to log.
type LogStringer interface {
LogString() string
}
// Holds a map[string]Level, mapping a package (field pkg in logs) to a log level.
// The empty string is the default/fallback log level.
var config atomic.Value
var lowestLevel atomic.Int32 // For quick initial check.
var config atomic.Pointer[map[string]slog.Level] // For secondary complete check for match.
func init() {
config.Store(map[string]Level{"": LevelError})
SetConfig(map[string]slog.Level{"": LevelInfo})
}
// SetConfig atomically sets the new log levels used by all Log instances.
func SetConfig(c map[string]Level) {
config.Store(c)
}
// Pair is a field/value pair, for use in logged lines.
type Pair struct {
Key string
Value any
}
// Field is a shorthand for making a Pair.
func Field(k string, v any) Pair {
return Pair{k, v}
}
// Log is an instance potentially with its own field/value pair added to any
// logging output.
type Log struct {
fields []Pair
moreFields func() []Pair
}
// New returns a new Log instance. Each log invocation adds field "pkg".
func New(pkg string) *Log {
return &Log{
fields: []Pair{{"pkg", pkg}},
func SetConfig(c map[string]slog.Level) {
lowest := c[""]
for _, l := range c {
if l < lowest {
lowest = l
}
}
lowestLevel.Store(int32(lowest))
config.Store(&c)
}
var (
// When the configured log level is any of the Trace levels, all protocol messages
// are printed. But protocol "data" (like an email message in the SMTP DATA
// command) is replaced with "..." unless the configured level is LevelTracedata.
// Likewise, protocol messages with authentication data (e.g. plaintext base64
// passwords) are replaced with "***" unless the configured level is LevelTraceauth
// or LevelTracedata.
LevelTracedata = slog.LevelDebug - 8
LevelTraceauth = slog.LevelDebug - 6
LevelTrace = slog.LevelDebug - 4
LevelDebug = slog.LevelDebug
LevelInfo = slog.LevelInfo
LevelError = slog.LevelError
LevelFatal = slog.LevelError + 4 // Printed regardless of configured log level.
LevelPrint = slog.LevelError + 8 // Printed regardless of configured log level.
)
// Levelstrings map log levels to human-readable names.
var LevelStrings = map[slog.Level]string{
LevelTracedata: "tracedata",
LevelTraceauth: "traceauth",
LevelTrace: "trace",
LevelDebug: "debug",
LevelInfo: "info",
LevelError: "error",
LevelFatal: "fatal",
LevelPrint: "print",
}
// Levels map the human-readable log level to a level.
var Levels = map[string]slog.Level{
"tracedata": LevelTracedata,
"traceauth": LevelTraceauth,
"trace": LevelTrace,
"debug": LevelDebug,
"info": LevelInfo,
"error": LevelError,
"fatal": LevelFatal,
"print": LevelPrint,
}
// Log wraps an slog.Logger, providing convenience functions.
type Log struct {
*slog.Logger
}
// New returns a Log that adds a "pkg" attribute. If logger is nil, a new
// Logger is created with a custom handler.
func New(pkg string, logger *slog.Logger) Log {
if logger == nil {
logger = slog.New(&handler{})
}
return Log{logger}.WithPkg(pkg)
}
// WithCid adds a attribute "cid".
// Also see WithContext.
func (l Log) WithCid(cid int64) Log {
return l.With(slog.Int64("cid", cid))
}
type key string
@ -124,19 +125,13 @@ type key string
// CidKey can be used with context.WithValue to store a "cid" in a context, for logging.
var CidKey key = "cid"
// WithCid adds a field "cid".
// Also see WithContext.
func (l *Log) WithCid(cid int64) *Log {
return l.Fields(Pair{"cid", cid})
}
// WithContext adds cid from context, if present. Context are often passed to
// functions, especially between packages, to pass a "cid" for an operation. At the
// start of a function (especially if exported) a variable "log" is often
// instantiated from a package-level variable "xlog", with WithContext for its cid.
// A *Log could be passed instead, but contexts are more pervasive. For the same
// instantiated from a package-level logger, with WithContext for its cid.
// Ideally, a Log could be passed instead, but contexts are more pervasive. For the same
// reason WithContext is more common than WithCid.
func (l *Log) WithContext(ctx context.Context) *Log {
func (l Log) WithContext(ctx context.Context) Log {
cidv := ctx.Value(CidKey)
if cidv == nil {
return l
@ -145,86 +140,200 @@ func (l *Log) WithContext(ctx context.Context) *Log {
return l.WithCid(cid)
}
// Field adds fields to the logger. Each logged line adds these fields.
func (l *Log) Fields(fields ...Pair) *Log {
nl := *l
nl.fields = append(fields, nl.fields...)
return &nl
// With adds attributes to to each logged line.
func (l Log) With(attrs ...slog.Attr) Log {
return Log{slog.New(l.Logger.Handler().WithAttrs(attrs))}
}
// MoreFields sets a function on the logger that is called just before logging,
// to retrieve additional fields to log.
func (l *Log) MoreFields(fn func() []Pair) *Log {
nl := *l
nl.moreFields = fn
return &nl
// WithPkg ensures pkg is added as attribute to logged lines. If the handler is
// an mlog handler, pkg is only added if not already the last added package.
func (l Log) WithPkg(pkg string) Log {
h := l.Logger.Handler()
if ph, ok := h.(*handler); ok {
if len(ph.Pkgs) > 0 && ph.Pkgs[len(ph.Pkgs)-1] == pkg {
return l
}
return Log{slog.New(ph.WithPkg(pkg))}
}
return Log{slog.New(h.WithAttrs([]slog.Attr{slog.String("pkg", pkg)}))}
}
// WithFunc sets fn to be called for additional attributes. Fn is only called
// when the line is logged.
// If the underlying handler is not an mlog.handler, this method has no effect.
// Caller must take care of preventing data races.
func (l Log) WithFunc(fn func() []slog.Attr) Log {
h := l.Logger.Handler()
if ph, ok := h.(*handler); ok {
return Log{slog.New(ph.WithFunc(fn))}
}
// Ignored for other handlers, only used internally (smtpserver, imapserver).
return l
}
// Check logs an error if err is not nil. Intended for logging errors that are good
// to know, but would not influence program flow.
func (l *Log) Check(err error, text string, fields ...Pair) {
func (l Log) Check(err error, msg string, attrs ...slog.Attr) {
if err != nil {
l.Errorx(text, err, fields...)
l.Errorx(msg, err, attrs...)
}
}
func (l *Log) Trace(traceLevel Level, text string) bool {
return l.logx(traceLevel, nil, text)
func errAttr(err error) slog.Attr {
return slog.Any("err", err)
}
func (l *Log) Fatal(text string, fields ...Pair) { l.Fatalx(text, nil, fields...) }
func (l *Log) Fatalx(text string, err error, fields ...Pair) {
l.plog(LevelFatal, err, text, fields...)
// todo: consider taking a context parameter. it would require all code be refactored. we may want to do this if callers really depend on passing attrs through context. the mox code base does not do that. it makes all call sites more tedious, and requires passing around ctx everywhere, so consider carefully.
func (l Log) Debug(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelDebug, msg, attrs...)
}
func (l Log) Debugx(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelDebug, msg, attrs...)
}
func (l Log) Info(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelInfo, msg, attrs...)
}
func (l Log) Infox(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelInfo, msg, attrs...)
}
func (l Log) Error(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelError, msg, attrs...)
}
func (l Log) Errorx(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelError, msg, attrs...)
}
func (l Log) Fatal(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelFatal, msg, attrs...)
os.Exit(1)
}
func (l *Log) Print(text string, fields ...Pair) bool {
return l.logx(LevelPrint, nil, text, fields...)
}
func (l *Log) Printx(text string, err error, fields ...Pair) bool {
return l.logx(LevelPrint, err, text, fields...)
func (l Log) Fatalx(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelFatal, msg, attrs...)
os.Exit(1)
}
func (l *Log) Debug(text string, fields ...Pair) bool {
return l.logx(LevelDebug, nil, text, fields...)
}
func (l *Log) Debugx(text string, err error, fields ...Pair) bool {
return l.logx(LevelDebug, err, text, fields...)
func (l Log) Print(msg string, attrs ...slog.Attr) {
l.Logger.LogAttrs(noctx, LevelPrint, msg, attrs...)
}
func (l *Log) Info(text string, fields ...Pair) bool { return l.logx(LevelInfo, nil, text, fields...) }
func (l *Log) Infox(text string, err error, fields ...Pair) bool {
return l.logx(LevelInfo, err, text, fields...)
func (l Log) Printx(msg string, err error, attrs ...slog.Attr) {
if err != nil {
attrs = append([]slog.Attr{errAttr(err)}, attrs...)
}
l.Logger.LogAttrs(noctx, LevelPrint, msg, attrs...)
}
func (l *Log) Error(text string, fields ...Pair) bool {
return l.logx(LevelError, nil, text, fields...)
}
func (l *Log) Errorx(text string, err error, fields ...Pair) bool {
return l.logx(LevelError, err, text, fields...)
}
// Trace logs at trace/traceauth/tracedata level.
// If the active log level is any of the trace levels, the data is logged.
// If level is for tracedata, but the active level doesn't trace data, data is replaced with "...".
// If level is for traceauth, but the active level doesn't trace auth, data is replaced with "***".
func (l Log) Trace(level slog.Level, prefix string, data []byte) {
h := l.Handler()
if !h.Enabled(noctx, level) {
return
}
ph, ok := h.(*handler)
if !ok {
msg := prefix + string(data)
r := slog.NewRecord(time.Now(), level, msg, 0)
h.Handle(noctx, r)
return
}
filterLevel, ok := ph.configMatch(level)
if !ok {
return
}
func (l *Log) logx(level Level, err error, text string, fields ...Pair) bool {
if ok, high := l.match(level); ok {
// Nothing.
} else if high >= LevelTrace && level == LevelTraceauth {
text = "***"
} else if high >= LevelTrace && level == LevelTracedata {
text = "..."
var msg string
if hideData, hideAuth := traceLevel(filterLevel, level); hideData {
msg = prefix + "..."
} else if hideAuth {
msg = prefix + "***"
} else {
return false
msg = prefix + string(data)
}
if level > LevelTrace {
level = LevelTrace
r := slog.NewRecord(time.Time{}, level, msg, 0)
ph.write(filterLevel, r)
}
func traceLevel(level, recordLevel slog.Level) (hideData, hideAuth bool) {
hideData = recordLevel == LevelTracedata && level > LevelTracedata
hideAuth = recordLevel == LevelTraceauth && level > LevelTraceauth
return
}
type handler struct {
Pkgs []string
Attrs []slog.Attr
Group string // Empty or with dot-separated names, ending with a dot.
Fn func() []slog.Attr // Only called when record is actually being logged.
}
func match(minLevel, level slog.Level) bool {
return level >= LevelFatal || level >= minLevel || minLevel <= LevelTrace && level <= LevelTrace
}
func (h *handler) Enabled(ctx context.Context, level slog.Level) bool {
return match(slog.Level(lowestLevel.Load()), level)
}
func (h *handler) configMatch(level slog.Level) (slog.Level, bool) {
c := *config.Load()
for i := len(h.Pkgs) - 1; i >= 0; i-- {
if l, ok := c[h.Pkgs[i]]; ok {
return l, match(l, level)
}
}
l := c[""]
return l, match(l, level)
}
func (h *handler) Handle(ctx context.Context, r slog.Record) error {
l, ok := h.configMatch(r.Level)
if !ok {
return nil
}
if hideData, hideAuth := traceLevel(l, r.Level); hideData {
r.Message = "..."
} else if hideAuth {
r.Message = "***"
}
return h.write(l, r)
}
// Reuse buffers to format log lines into.
var logBuffersStore [32][256]byte
var logBuffers = make(chan []byte, 200)
func init() {
for i := range logBuffersStore {
logBuffers <- logBuffersStore[i][:]
}
l.plog(level, err, text, fields...)
return true
}
// escape logfmt string if required, otherwise return original string.
func logfmtValue(s string) string {
func formatString(s string) string {
for _, c := range s {
if c == '"' || c == '\\' || c <= ' ' || c == '=' || c >= 0x7f {
if c <= ' ' || c == '"' || c == '\\' || c == '=' || c >= 0x7f {
return fmt.Sprintf("%q", s)
}
}
@ -261,6 +370,8 @@ func stringValue(iscid, nested bool, v any) string {
return ""
}
return "[" + strings.Join(r, ",") + "]"
case error:
return r.Error()
}
rv := reflect.ValueOf(v)
@ -320,100 +431,188 @@ func stringValue(iscid, nested bool, v any) string {
}
first = false
k := strings.ToLower(t.Field(i).Name)
b.WriteString(k + "=" + logfmtValue(vs))
b.WriteString(k + "=" + vs)
}
return b.String()
}
func (l *Log) plog(level Level, err error, text string, fields ...Pair) {
fields = append(l.fields, fields...)
if l.moreFields != nil {
fields = append(fields, l.moreFields()...)
func writeAttr(w io.Writer, separator, group string, a slog.Attr) {
switch a.Value.Kind() {
case slog.KindGroup:
if group != "" {
group += "."
}
group += a.Key
for _, a := range a.Value.Group() {
writeAttr(w, separator, group, a)
}
return
default:
var vv any
if a.Value.Kind() == slog.KindLogValuer {
vv = a.Value.Resolve().Any()
} else {
vv = a.Value.Any()
}
s := stringValue(a.Key == "cid", false, vv)
fmt.Fprint(w, separator, group, a.Key, "=", formatString(s))
}
// We build up a buffer so we can do a single atomic write of the data. Otherwise partial log lines may interleaf.
b := &bytes.Buffer{}
if Logfmt {
fmt.Fprintf(b, "l=%s m=%s", LevelStrings[level], logfmtValue(text))
if err != nil {
fmt.Fprintf(b, " err=%s", logfmtValue(err.Error()))
}
for i := 0; i < len(fields); i++ {
kv := fields[i]
fmt.Fprintf(b, " %s=%s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
}
b.WriteString("\n")
} else {
fmt.Fprintf(b, "%s: %s", LevelStrings[level], logfmtValue(text))
if err != nil {
fmt.Fprintf(b, ": %s", logfmtValue(err.Error()))
}
if len(fields) > 0 {
fmt.Fprint(b, " (")
for i := 0; i < len(fields); i++ {
if i > 0 {
fmt.Fprint(b, "; ")
}
kv := fields[i]
fmt.Fprintf(b, "%s: %s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
}
fmt.Fprint(b, ")")
}
b.WriteString("\n")
}
os.Stderr.Write(b.Bytes())
}
func (l *Log) match(level Level) (bool, Level) {
if level == LevelPrint || level == LevelFatal {
return true, level
func (h *handler) write(l slog.Level, r slog.Record) error {
// Reuse a buffer, or temporarily allocate a new one.
var buf []byte
select {
case buf = <-logBuffers:
defer func() {
logBuffers <- buf
}()
default:
buf = make([]byte, 128)
}
cl := config.Load().(map[string]Level)
b := bytes.NewBuffer(buf[:0])
eb := &errWriter{b, nil}
seen := false
var high Level
for _, kv := range l.fields {
if kv.Key != "pkg" {
continue
if Logfmt {
var wrotePkgs bool
ensurePkgs := func() {
if !wrotePkgs {
wrotePkgs = true
for _, pkg := range h.Pkgs {
writeAttr(eb, " ", "", slog.String("pkg", pkg))
}
}
}
pkg, ok := kv.Value.(string)
if !ok {
continue
fmt.Fprint(eb, "l=", LevelStrings[r.Level], " m=")
fmt.Fprintf(eb, "%q", r.Message)
n := 0
r.Attrs(func(a slog.Attr) bool {
if n > 0 || a.Key != "err" || h.Group != "" {
ensurePkgs()
}
writeAttr(eb, " ", h.Group, a)
n++
return true
})
ensurePkgs()
for _, a := range h.Attrs {
writeAttr(eb, " ", h.Group, a)
}
v, ok := cl[pkg]
if v > high {
high = v
if h.Fn != nil {
for _, a := range h.Fn() {
writeAttr(eb, " ", h.Group, a)
}
}
if ok && v >= level {
return true, high
fmt.Fprint(eb, "\n")
} else {
var wrotePkgs bool
ensurePkgs := func() {
if !wrotePkgs {
wrotePkgs = true
for _, pkg := range h.Pkgs {
writeAttr(eb, "; ", "", slog.String("pkg", pkg))
}
}
}
seen = seen || ok
fmt.Fprint(eb, LevelStrings[r.Level], ": ", r.Message)
n := 0
r.Attrs(func(a slog.Attr) bool {
if n == 0 && a.Key == "err" && h.Group == "" {
fmt.Fprint(eb, ": ", a.Value.String())
ensurePkgs()
} else {
ensurePkgs()
writeAttr(eb, "; ", h.Group, a)
}
n++
return true
})
ensurePkgs()
for _, a := range h.Attrs {
writeAttr(eb, "; ", h.Group, a)
}
if h.Fn != nil {
for _, a := range h.Fn() {
writeAttr(eb, "; ", h.Group, a)
n++
}
}
fmt.Fprint(eb, "\n")
}
if seen {
return false, high
if eb.Err != nil {
return eb.Err
}
v, ok := cl[""]
if v > high {
high = v
}
return ok && v >= level, v
// todo: for mox serve, do writes in separate goroutine.
_, err := os.Stderr.Write(b.Bytes())
return err
}
type errWriter struct {
log *Log
level Level
msg string
Writer *bytes.Buffer
Err error
}
func (w *errWriter) Write(buf []byte) (int, error) {
err := errors.New(strings.TrimSpace(string(buf)))
w.log.logx(w.level, err, w.msg)
if w.Err != nil {
return 0, w.Err
}
var n int
n, w.Err = w.Writer.Write(buf)
return n, w.Err
}
func (h *handler) WithAttrs(attrs []slog.Attr) slog.Handler {
nh := *h
if h.Attrs != nil {
nh.Attrs = append([]slog.Attr{}, h.Attrs...)
}
nh.Attrs = append(nh.Attrs, attrs...)
return &nh
}
func (h *handler) WithGroup(name string) slog.Handler {
if name == "" {
return h
}
nh := *h
nh.Group += name + "."
return &nh
}
func (h *handler) WithPkg(pkg string) *handler {
nh := *h
if nh.Pkgs != nil {
nh.Pkgs = append([]string{}, nh.Pkgs...)
}
nh.Pkgs = append(nh.Pkgs, pkg)
return &nh
}
func (h *handler) WithFunc(fn func() []slog.Attr) *handler {
nh := *h
nh.Fn = fn
return &nh
}
type logWriter struct {
log Log
level slog.Level
msg string
}
func (w logWriter) Write(buf []byte) (int, error) {
err := strings.TrimSpace(string(buf))
w.log.LogAttrs(noctx, w.level, w.msg, slog.String("err", err))
return len(buf), nil
}
// ErrWriter returns a writer that turns each write into a logging call on "log"
// LogWriter returns a writer that turns each write into a logging call on "log"
// with given "level" and "msg" and the written content as an error.
// Can be used for making a Go log.Logger for use in http.Server.ErrorLog.
func ErrWriter(log *Log, level Level, msg string) io.Writer {
return &errWriter{log, level, msg}
func LogWriter(log Log, level slog.Level, msg string) io.Writer {
return logWriter{log, level, msg}
}

72
mox-/admin.go
View file

@ -20,6 +20,7 @@ import (
"time"
"golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/mjl-/adns"
@ -28,7 +29,6 @@ import (
"github.com/mjl-/mox/dmarc"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/junk"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mtasts"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/tlsrpt"
@ -154,7 +154,7 @@ func MakeAccountConfig(addr smtp.Address) config.Account {
// MakeDomainConfig makes a new config for a domain, creating DKIM keys, using
// accountName for DMARC and TLS reports.
func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountName string, withMTASTS bool) (config.Domain, []string, error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
now := time.Now()
year := now.Format("2006")
@ -164,7 +164,7 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN
defer func() {
for _, p := range paths {
err := os.Remove(p)
log.Check(err, "removing path for domain config", mlog.Field("path", p))
log.Check(err, "removing path for domain config", slog.String("path", p))
}
}()
@ -180,7 +180,7 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN
err := f.Close()
log.Check(err, "closing file after error")
err = os.Remove(path)
log.Check(err, "removing file after error", mlog.Field("path", path))
log.Check(err, "removing file after error", slog.String("path", path))
}
}()
if _, err := f.Write(data); err != nil {
@ -288,10 +288,10 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN
// If the account does not exist, it is created with localpart. Localpart must be
// set only if the account does not yet exist.
func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, localpart smtp.Localpart) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("adding domain", rerr, mlog.Field("domain", domain), mlog.Field("account", accountName), mlog.Field("localpart", localpart))
log.Errorx("adding domain", rerr, slog.Any("domain", domain), slog.String("account", accountName), slog.Any("localpart", localpart))
}
}()
@ -327,7 +327,7 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local
defer func() {
for _, f := range cleanupFiles {
err := os.Remove(f)
log.Check(err, "cleaning up file after error", mlog.Field("path", f))
log.Check(err, "cleaning up file after error", slog.String("path", f))
}
}()
@ -356,7 +356,7 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("domain added", mlog.Field("domain", domain))
log.Info("domain added", slog.Any("domain", domain))
cleanupFiles = nil // All good, don't cleanup.
return nil
}
@ -365,10 +365,10 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local
//
// No accounts are removed, also not when they still reference this domain.
func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("removing domain", rerr, mlog.Field("domain", domain))
log.Errorx("removing domain", rerr, slog.Any("domain", domain))
}
}()
@ -418,16 +418,16 @@ func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) {
err = os.Rename(src, dst)
}
if err != nil {
log.Errorx("renaming dkim private key file for removed domain", err, mlog.Field("src", src), mlog.Field("dst", dst))
log.Errorx("renaming dkim private key file for removed domain", err, slog.String("src", src), slog.String("dst", dst))
}
}
log.Info("domain removed", mlog.Field("domain", domain))
log.Info("domain removed", slog.Any("domain", domain))
return nil
}
func WebserverConfigSet(ctx context.Context, domainRedirects map[string]string, webhandlers []config.WebHandler) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("saving webserver config", rerr)
@ -680,10 +680,10 @@ func DomainRecords(domConf config.Domain, domain dns.Domain, hasDNSSEC bool) ([]
//
// Catchall addresses are not supported for AccountAdd. Add separately with AddressAdd.
func AccountAdd(ctx context.Context, account, address string) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("adding account", rerr, mlog.Field("account", account), mlog.Field("address", address))
log.Errorx("adding account", rerr, slog.String("account", account), slog.String("address", address))
}
}()
@ -716,16 +716,16 @@ func AccountAdd(ctx context.Context, account, address string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("account added", mlog.Field("account", account), mlog.Field("address", addr))
log.Info("account added", slog.String("account", account), slog.Any("address", addr))
return nil
}
// AccountRemove removes an account and reloads the configuration.
func AccountRemove(ctx context.Context, account string) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("adding account", rerr, mlog.Field("account", account))
log.Errorx("adding account", rerr, slog.String("account", account))
}
}()
@ -750,7 +750,7 @@ func AccountRemove(ctx context.Context, account string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("account removed", mlog.Field("account", account))
log.Info("account removed", slog.String("account", account))
return nil
}
@ -775,10 +775,10 @@ func checkAddressAvailable(addr smtp.Address) error {
// AddressAdd adds an email address to an account and reloads the configuration. If
// address starts with an @ it is treated as a catchall address for the domain.
func AddressAdd(ctx context.Context, address, account string) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("adding address", rerr, mlog.Field("address", address), mlog.Field("account", account))
log.Errorx("adding address", rerr, slog.String("address", address), slog.String("account", account))
}
}()
@ -834,16 +834,16 @@ func AddressAdd(ctx context.Context, address, account string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("address added", mlog.Field("address", address), mlog.Field("account", account))
log.Info("address added", slog.String("address", address), slog.String("account", account))
return nil
}
// AddressRemove removes an email address and reloads the configuration.
func AddressRemove(ctx context.Context, address string) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("removing address", rerr, mlog.Field("address", address))
log.Errorx("removing address", rerr, slog.String("address", address))
}
}()
@ -884,16 +884,16 @@ func AddressRemove(ctx context.Context, address string) (rerr error) {
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("address removed", mlog.Field("address", address), mlog.Field("account", ad.Account))
log.Info("address removed", slog.String("address", address), slog.String("account", ad.Account))
return nil
}
// AccountFullNameSave updates the full name for an account and reloads the configuration.
func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("saving account full name", rerr, mlog.Field("account", account))
log.Errorx("saving account full name", rerr, slog.String("account", account))
}
}()
@ -920,16 +920,16 @@ func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr er
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("account full name saved", mlog.Field("account", account))
log.Info("account full name saved", slog.String("account", account))
return nil
}
// DestinationSave updates a destination for an account and reloads the configuration.
func DestinationSave(ctx context.Context, account, destName string, newDest config.Destination) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("saving destination", rerr, mlog.Field("account", account), mlog.Field("destname", destName), mlog.Field("destination", newDest))
log.Errorx("saving destination", rerr, slog.String("account", account), slog.String("destname", destName), slog.Any("destination", newDest))
}
}()
@ -965,16 +965,16 @@ func DestinationSave(ctx context.Context, account, destName string, newDest conf
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("destination saved", mlog.Field("account", account), mlog.Field("destname", destName))
log.Info("destination saved", slog.String("account", account), slog.String("destname", destName))
return nil
}
// AccountLimitsSave saves new message sending limits for an account.
func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesPerDay, maxFirstTimeRecipientsPerDay int) (rerr error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
defer func() {
if rerr != nil {
log.Errorx("saving account limits", rerr, mlog.Field("account", account))
log.Errorx("saving account limits", rerr, slog.String("account", account))
}
}()
@ -1001,7 +1001,7 @@ func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesP
if err := writeDynamic(ctx, log, nc); err != nil {
return fmt.Errorf("writing domains.conf: %v", err)
}
log.Info("account limits saved", mlog.Field("account", account))
log.Info("account limits saved", slog.String("account", account))
return nil
}
@ -1157,7 +1157,7 @@ func ClientConfigsDomain(d dns.Domain) (ClientConfigs, error) {
// IPs returns ip addresses we may be listening/receiving mail on or
// connecting/sending from to the outside.
func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) {
log := xlog.WithContext(ctx)
log := pkglog.WithContext(ctx)
// Try to gather all IPs we are listening on by going through the config.
// If we encounter 0.0.0.0 or ::, we'll gather all local IPs afterwards.
@ -1208,7 +1208,7 @@ func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) {
for _, addr := range addrs {
ip, _, err := net.ParseCIDR(addr.String())
if err != nil {
log.Errorx("bad interface addr", err, mlog.Field("address", addr))
log.Errorx("bad interface addr", err, slog.Any("address", addr))
continue
}
v4 := ip.To4() != nil

95
mox-/config.go
View file

@ -28,6 +28,7 @@ import (
"sync"
"time"
"golang.org/x/exp/slog"
"golang.org/x/text/unicode/norm"
"github.com/mjl-/autocert"
@ -44,14 +45,14 @@ import (
"github.com/mjl-/mox/smtp"
)
var xlog = mlog.New("mox")
var pkglog = mlog.New("mox", nil)
// Config paths are set early in program startup. They will point to files in
// the same directory.
var (
ConfigStaticPath string
ConfigDynamicPath string
Conf = Config{Log: map[string]mlog.Level{"": mlog.LevelError}}
Conf = Config{Log: map[string]slog.Level{"": slog.LevelError}}
)
// Config as used in the code, a processed version of what is in the config file.
@ -61,7 +62,7 @@ type Config struct {
Static config.Static // Does not change during the lifetime of a running instance.
logMutex sync.Mutex // For accessing the log levels.
Log map[string]mlog.Level
Log map[string]slog.Level
dynamicMutex sync.Mutex
Dynamic config.Dynamic // Can only be accessed directly by tests. Use methods on Config for locked access.
@ -83,31 +84,31 @@ type AccountDestination struct {
// LogLevelSet sets a new log level for pkg. An empty pkg sets the default log
// value that is used if no explicit log level is configured for a package.
// This change is ephemeral, no config file is changed.
func (c *Config) LogLevelSet(pkg string, level mlog.Level) {
func (c *Config) LogLevelSet(log mlog.Log, pkg string, level slog.Level) {
c.logMutex.Lock()
defer c.logMutex.Unlock()
l := c.copyLogLevels()
l[pkg] = level
c.Log = l
xlog.Print("log level changed", mlog.Field("pkg", pkg), mlog.Field("level", mlog.LevelStrings[level]))
log.Print("log level changed", slog.String("pkg", pkg), slog.Any("level", mlog.LevelStrings[level]))
mlog.SetConfig(c.Log)
}
// LogLevelRemove removes a configured log level for a package.
func (c *Config) LogLevelRemove(pkg string) {
func (c *Config) LogLevelRemove(log mlog.Log, pkg string) {
c.logMutex.Lock()
defer c.logMutex.Unlock()
l := c.copyLogLevels()
delete(l, pkg)
c.Log = l
xlog.Print("log level cleared", mlog.Field("pkg", pkg))
log.Print("log level cleared", slog.String("pkg", pkg))
mlog.SetConfig(c.Log)
}
// copyLogLevels returns a copy of c.Log, for modifications.
// must be called with log lock held.
func (c *Config) copyLogLevels() map[string]mlog.Level {
m := map[string]mlog.Level{}
func (c *Config) copyLogLevels() map[string]slog.Level {
m := map[string]slog.Level{}
for pkg, level := range c.Log {
m[pkg] = level
}
@ -115,7 +116,7 @@ func (c *Config) copyLogLevels() map[string]mlog.Level {
}
// LogLevels returns a copy of the current log levels.
func (c *Config) LogLevels() map[string]mlog.Level {
func (c *Config) LogLevels() map[string]slog.Level {
c.logMutex.Lock()
defer c.logMutex.Unlock()
return c.copyLogLevels()
@ -128,12 +129,12 @@ func (c *Config) withDynamicLock(fn func()) {
if now.Sub(c.DynamicLastCheck) > time.Second {
c.DynamicLastCheck = now
if fi, err := os.Stat(ConfigDynamicPath); err != nil {
xlog.Errorx("stat domains config", err)
pkglog.Errorx("stat domains config", err)
} else if !fi.ModTime().Equal(c.dynamicMtime) {
if errs := c.loadDynamic(); len(errs) > 0 {
xlog.Errorx("loading domains config", errs[0], mlog.Field("errors", errs))
pkglog.Errorx("loading domains config", errs[0], slog.Any("errors", errs))
} else {
xlog.Info("domains config reloaded")
pkglog.Info("domains config reloaded")
c.dynamicMtime = fi.ModTime()
}
}
@ -143,14 +144,14 @@ func (c *Config) withDynamicLock(fn func()) {
// must be called with dynamic lock held.
func (c *Config) loadDynamic() []error {
d, mtime, accDests, err := ParseDynamicConfig(context.Background(), ConfigDynamicPath, c.Static)
d, mtime, accDests, err := ParseDynamicConfig(context.Background(), pkglog, ConfigDynamicPath, c.Static)
if err != nil {
return err
}
c.Dynamic = d
c.dynamicMtime = mtime
c.accountDestinations = accDests
c.allowACMEHosts(true)
c.allowACMEHosts(pkglog, true)
return nil
}
@ -236,7 +237,7 @@ func (c *Config) Routes(accountName string, domain dns.Domain) (accountRoutes, d
return
}
func (c *Config) allowACMEHosts(checkACMEHosts bool) {
func (c *Config) allowACMEHosts(log mlog.Log, checkACMEHosts bool) {
for _, l := range c.Static.Listeners {
if l.TLS == nil || l.TLS.ACME == "" {
continue
@ -259,7 +260,7 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) {
if l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS {
if d, err := dns.ParseDomain("autoconfig." + dom.Domain.ASCII); err != nil {
xlog.Errorx("parsing autoconfig domain", err, mlog.Field("domain", dom.Domain))
log.Errorx("parsing autoconfig domain", err, slog.Any("domain", dom.Domain))
} else {
hostnames[d] = struct{}{}
}
@ -268,7 +269,7 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) {
if l.MTASTSHTTPS.Enabled && dom.MTASTS != nil && !l.MTASTSHTTPS.NonTLS {
d, err := dns.ParseDomain("mta-sts." + dom.Domain.ASCII)
if err != nil {
xlog.Errorx("parsing mta-sts domain", err, mlog.Field("domain", dom.Domain))
log.Errorx("parsing mta-sts domain", err, slog.Any("domain", dom.Domain))
} else {
hostnames[d] = struct{}{}
}
@ -292,15 +293,15 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) {
if public.IPsNATed {
ips = nil
}
m.SetAllowedHostnames(dns.StrictResolver{Pkg: "autotls"}, hostnames, ips, checkACMEHosts)
m.SetAllowedHostnames(log, dns.StrictResolver{Pkg: "autotls", Log: log.Logger}, hostnames, ips, checkACMEHosts)
}
}
// todo future: write config parsing & writing code that can read a config and remembers the exact tokens including newlines and comments, and can write back a modified file. the goal is to be able to write a config file automatically (after changing fields through the ui), but not loose comments and whitespace, to still get useful diffs for storing the config in a version control system.
// must be called with lock held.
func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error {
accDests, errs := prepareDynamicConfig(ctx, ConfigDynamicPath, Conf.Static, &c)
func writeDynamic(ctx context.Context, log mlog.Log, c config.Dynamic) error {
accDests, errs := prepareDynamicConfig(ctx, log, ConfigDynamicPath, Conf.Static, &c)
if len(errs) > 0 {
return errs[0]
}
@ -330,7 +331,7 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error {
if err := f.Sync(); err != nil {
return fmt.Errorf("sync domains.conf after write: %v", err)
}
if err := moxio.SyncDir(filepath.Dir(ConfigDynamicPath)); err != nil {
if err := moxio.SyncDir(log, filepath.Dir(ConfigDynamicPath)); err != nil {
return fmt.Errorf("sync dir of domains.conf after write: %v", err)
}
@ -349,32 +350,32 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error {
Conf.Dynamic = c
Conf.accountDestinations = accDests
Conf.allowACMEHosts(true)
Conf.allowACMEHosts(log, true)
return nil
}
// MustLoadConfig loads the config, quitting on errors.
func MustLoadConfig(doLoadTLSKeyCerts, checkACMEHosts bool) {
errs := LoadConfig(context.Background(), doLoadTLSKeyCerts, checkACMEHosts)
errs := LoadConfig(context.Background(), pkglog, doLoadTLSKeyCerts, checkACMEHosts)
if len(errs) > 1 {
xlog.Error("loading config file: multiple errors")
pkglog.Error("loading config file: multiple errors")
for _, err := range errs {
xlog.Errorx("config error", err)
pkglog.Errorx("config error", err)
}
xlog.Fatal("stopping after multiple config errors")
pkglog.Fatal("stopping after multiple config errors")
} else if len(errs) == 1 {
xlog.Fatalx("loading config file", errs[0])
pkglog.Fatalx("loading config file", errs[0])
}
}
// LoadConfig attempts to parse and load a config, returning any errors
// encountered.
func LoadConfig(ctx context.Context, doLoadTLSKeyCerts, checkACMEHosts bool) []error {
func LoadConfig(ctx context.Context, log mlog.Log, doLoadTLSKeyCerts, checkACMEHosts bool) []error {
Shutdown, ShutdownCancel = context.WithCancel(context.Background())
Context, ContextCancel = context.WithCancel(context.Background())
c, errs := ParseConfig(ctx, ConfigStaticPath, false, doLoadTLSKeyCerts, checkACMEHosts)
c, errs := ParseConfig(ctx, log, ConfigStaticPath, false, doLoadTLSKeyCerts, checkACMEHosts)
if len(errs) > 0 {
return errs
}
@ -405,7 +406,7 @@ func SetConfig(c *Config) {
// quickstart in the case the user is going to provide their own certificates.
// If checkACMEHosts is true, the hosts allowed for acme are compared with the
// explicitly configured ips we are listening on.
func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, checkACMEHosts bool) (c *Config, errs []error) {
func ParseConfig(ctx context.Context, log mlog.Log, p string, checkOnly, doLoadTLSKeyCerts, checkACMEHosts bool) (c *Config, errs []error) {
c = &Config{
Static: config.Static{
DataDir: ".",
@ -424,15 +425,15 @@ func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, ch
return nil, []error{fmt.Errorf("parsing %s%v", p, err)}
}
if xerrs := PrepareStaticConfig(ctx, p, c, checkOnly, doLoadTLSKeyCerts); len(xerrs) > 0 {
if xerrs := PrepareStaticConfig(ctx, log, p, c, checkOnly, doLoadTLSKeyCerts); len(xerrs) > 0 {
return nil, xerrs
}
pp := filepath.Join(filepath.Dir(p), "domains.conf")
c.Dynamic, c.dynamicMtime, c.accountDestinations, errs = ParseDynamicConfig(ctx, pp, c.Static)
c.Dynamic, c.dynamicMtime, c.accountDestinations, errs = ParseDynamicConfig(ctx, log, pp, c.Static)
if !checkOnly {
c.allowACMEHosts(checkACMEHosts)
c.allowACMEHosts(log, checkACMEHosts)
}
return c, errs
@ -441,13 +442,11 @@ func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, ch
// PrepareStaticConfig parses the static config file and prepares data structures
// for starting mox. If checkOnly is set no substantial changes are made, like
// creating an ACME registration.
func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, checkOnly, doLoadTLSKeyCerts bool) (errs []error) {
func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, conf *Config, checkOnly, doLoadTLSKeyCerts bool) (errs []error) {
addErrorf := func(format string, args ...any) {
errs = append(errs, fmt.Errorf(format, args...))
}
log := xlog.WithContext(ctx)
c := &conf.Static
// check that mailbox is in unicode NFC normalized form.
@ -461,7 +460,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
// Post-process logging config.
if logLevel, ok := mlog.Levels[c.LogLevel]; ok {
conf.Log = map[string]mlog.Level{"": logLevel}
conf.Log = map[string]slog.Level{"": logLevel}
} else {
addErrorf("invalid log level %q", c.LogLevel)
}
@ -569,10 +568,10 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
key = findACMEHostPrivateKey(acmeName, host, keyType, 2)
}
if key != nil {
log.Debug("found existing private key for certificate for host", mlog.Field("acmename", acmeName), mlog.Field("host", host), mlog.Field("keytype", keyType))
log.Debug("found existing private key for certificate for host", slog.String("acmename", acmeName), slog.String("host", host), slog.Any("keytype", keyType))
return key, nil
}
log.Debug("generating new private key for certificate for host", mlog.Field("acmename", acmeName), mlog.Field("host", host), mlog.Field("keytype", keyType))
log.Debug("generating new private key for certificate for host", slog.String("acmename", acmeName), slog.String("host", host), slog.Any("keytype", keyType))
switch keyType {
case autocert.KeyRSA2048:
return rsa.GenerateKey(cryptorand.Reader, 2048)
@ -658,18 +657,18 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
switch k := privKey.(type) {
case *rsa.PrivateKey:
if k.N.BitLen() != 2048 {
log.Error("need rsa key with 2048 bits, for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath), mlog.Field("bits", k.N.BitLen()))
log.Error("need rsa key with 2048 bits, for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath), slog.Int("bits", k.N.BitLen()))
continue
}
l.TLS.HostPrivateRSA2048Keys = append(l.TLS.HostPrivateRSA2048Keys, k)
case *ecdsa.PrivateKey:
if k.Curve != elliptic.P256() {
log.Error("unrecognized ecdsa curve for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath))
log.Error("unrecognized ecdsa curve for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath))
continue
}
l.TLS.HostPrivateECDSAP256Keys = append(l.TLS.HostPrivateECDSAP256Keys, k)
default:
log.Error("unrecognized key type for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath), mlog.Field("keytype", fmt.Sprintf("%T", privKey)))
log.Error("unrecognized key type for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath), slog.String("keytype", fmt.Sprintf("%T", privKey)))
continue
}
}
@ -914,7 +913,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c
}
// PrepareDynamicConfig parses the dynamic config file given a static file.
func ParseDynamicConfig(ctx context.Context, dynamicPath string, static config.Static) (c config.Dynamic, mtime time.Time, accDests map[string]AccountDestination, errs []error) {
func ParseDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, static config.Static) (c config.Dynamic, mtime time.Time, accDests map[string]AccountDestination, errs []error) {
addErrorf := func(format string, args ...any) {
errs = append(errs, fmt.Errorf(format, args...))
}
@ -934,13 +933,11 @@ func ParseDynamicConfig(ctx context.Context, dynamicPath string, static config.S
return
}
accDests, errs = prepareDynamicConfig(ctx, dynamicPath, static, &c)
accDests, errs = prepareDynamicConfig(ctx, log, dynamicPath, static, &c)
return c, fi.ModTime(), accDests, errs
}
func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config.Static, c *config.Dynamic) (accDests map[string]AccountDestination, errs []error) {
log := xlog.WithContext(ctx)
func prepareDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, static config.Static, c *config.Dynamic) (accDests map[string]AccountDestination, errs []error) {
addErrorf := func(format string, args ...any) {
errs = append(errs, fmt.Errorf(format, args...))
}
@ -1321,7 +1318,7 @@ func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config
if !ok {
addErrorf("could not find localpart %q to replace with address in destinations", lp)
} else {
log.Error(`deprecation warning: support for account destination addresses specified as just localpart ("username") instead of full email address will be removed in the future; update domains.conf, for each Account, for each Destination, ensure each key is an email address by appending "@" and the default domain for the account`, mlog.Field("localpart", lp), mlog.Field("address", addr), mlog.Field("account", accName))
log.Error(`deprecation warning: support for account destination addresses specified as just localpart ("username") instead of full email address will be removed in the future; update domains.conf, for each Account, for each Destination, ensure each key is an email address by appending "@" and the default domain for the account`, slog.Any("localpart", lp), slog.Any("address", addr), slog.String("account", accName))
acc.Destinations[addr] = dest
delete(acc.Destinations, lp)
}

10
mox-/forkexec_unix.go
View file

@ -8,7 +8,7 @@ import (
"strings"
"syscall"
"github.com/mjl-/mox/mlog"
"golang.org/x/exp/slog"
)
// Fork and exec as unprivileged user.
@ -19,7 +19,7 @@ import (
func ForkExecUnprivileged() {
prog, err := os.Executable()
if err != nil {
xlog.Fatalx("finding executable for exec", err)
pkglog.Fatalx("finding executable for exec", err)
}
files := []*os.File{os.Stdin, os.Stdout, os.Stderr}
@ -49,7 +49,7 @@ func ForkExecUnprivileged() {
},
})
if err != nil {
xlog.Fatalx("fork and exec", err)
pkglog.Fatalx("fork and exec", err)
}
CleanupPassedFiles()
@ -66,9 +66,9 @@ func ForkExecUnprivileged() {
st, err := p.Wait()
if err != nil {
xlog.Fatalx("wait", err)
pkglog.Fatalx("wait", err)
}
code := st.ExitCode()
xlog.Print("stopping after child exit", mlog.Field("exitcode", code))
pkglog.Print("stopping after child exit", slog.Int("exitcode", code))
os.Exit(code)
}

10
mox-/lifecycle.go
View file

@ -32,7 +32,7 @@ func RestorePassedFiles() {
if runtime.GOOS == "linux" {
linuxhint = " If you updated from v0.0.1, update the mox.service file to start as root (privileges are dropped): ./mox config printservice >mox.service && sudo systemctl daemon-reload && sudo systemctl restart mox."
}
xlog.Fatal("mox must be started as root, and will drop privileges after binding required sockets (missing environment variable MOX_SOCKETS)." + linuxhint)
pkglog.Fatal("mox must be started as root, and will drop privileges after binding required sockets (missing environment variable MOX_SOCKETS)." + linuxhint)
}
// 0,1,2 are stdin,stdout,stderr, 3 is the first passed fd (first listeners, then files).
@ -59,12 +59,12 @@ func RestorePassedFiles() {
func CleanupPassedFiles() {
for _, f := range passedListeners {
err := f.Close()
xlog.Check(err, "closing listener socket file descriptor")
pkglog.Check(err, "closing listener socket file descriptor")
}
for _, fl := range passedFiles {
for _, f := range fl {
err := f.Close()
xlog.Check(err, "closing path file descriptor")
pkglog.Check(err, "closing path file descriptor")
}
}
}
@ -193,7 +193,7 @@ func (c *connections) Register(nc net.Conn, protocol, listener string) {
// doesn't hurt to log it.
select {
case <-Shutdown.Done():
xlog.Error("new connection added while shutting down")
pkglog.Error("new connection added while shutting down")
debug.PrintStack()
default:
}
@ -258,7 +258,7 @@ func (c *connections) Shutdown() {
defer c.Unlock()
for nc := range c.conns {
if err := nc.SetDeadline(now); err != nil {
xlog.Errorx("setting immediate read/write deadline for shutdown", err)
pkglog.Errorx("setting immediate read/write deadline for shutdown", err)
}
}
}

8
message/tlsrecv.go → mox-/tlsrecv.go
View file

@ -1,14 +1,16 @@
package message
package mox
import (
"crypto/tls"
"fmt"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog"
)
// TLSReceivedComment returns a comment about TLS of the connection for use in a Receive header.
func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string {
func TLSReceivedComment(log mlog.Log, cs tls.ConnectionState) []string {
// todo future: we could use the "tls" clause for the Received header as specified in ../rfc/8314:496. however, the text implies it is only for submission, not regular smtp. and it cannot specify the tls version. for now, not worth the trouble.
// Comments from other mail servers:
@ -32,7 +34,7 @@ func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string {
if version, ok := versions[cs.Version]; ok {
add(version)
} else {
log.Info("unknown tls version identifier", mlog.Field("version", cs.Version))
log.Info("unknown tls version identifier", slog.Any("version", cs.Version))
add(fmt.Sprintf("TLS identifier %x", cs.Version))
}

12
moxio/bufpool.go
View file

@ -6,11 +6,11 @@ import (
"fmt"
"io"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog"
)
var xlog = mlog.New("moxio")
// todo: instead of a bufpool, should maybe just make an alternative to bufio.Reader with a big enough buffer that we can fully use to read a line.
var ErrLineTooLong = errors.New("line from remote too long") // Returned by Bufpool.Readline.
@ -49,9 +49,9 @@ func (b *Bufpool) get() []byte {
// be all the bytes that have been read in the buffer. If the pool is full, the
// buffer is discarded, and will be cleaned up by the garbage collector.
// The caller should no longer reference "buf" after a call to put.
func (b *Bufpool) put(buf []byte, n int) {
func (b *Bufpool) put(log mlog.Log, buf []byte, n int) {
if len(buf) != b.size {
xlog.Error("buffer with bad size returned, ignoring", mlog.Field("badsize", len(buf)), mlog.Field("expsize", b.size))
log.Error("buffer with bad size returned, ignoring", slog.Int("badsize", len(buf)), slog.Int("expsize", b.size))
return
}
@ -67,11 +67,11 @@ func (b *Bufpool) put(buf []byte, n int) {
// Readline reads a \n- or \r\n-terminated line. Line is returned without \n or \r\n.
// If the line was too long, ErrLineTooLong is returned.
// If an EOF is encountered before a \n, io.ErrUnexpectedEOF is returned.
func (b *Bufpool) Readline(r *bufio.Reader) (line string, rerr error) {
func (b *Bufpool) Readline(log mlog.Log, r *bufio.Reader) (line string, rerr error) {
var nread int
buf := b.get()
defer func() {
b.put(buf, nread)
b.put(log, buf, nread)
}()
// Read until newline. If we reach the end of the buffer first, we write back an

17
moxio/bufpool_test.go
View file

@ -7,6 +7,8 @@ import (
"io"
"strings"
"testing"
"github.com/mjl-/mox/mlog"
)
func TestBufpool(t *testing.T) {
@ -16,8 +18,9 @@ func TestBufpool(t *testing.T) {
for i := 0; i < len(a); i++ {
a[i] = 1
}
bp.put(a, len(a)) // Will be stored.
bp.put(b, 0) // Will be discarded.
log := mlog.New("moxio", nil)
bp.put(log, a, len(a)) // Will be stored.
bp.put(log, b, 0) // Will be discarded.
na := bp.get()
if fmt.Sprintf("%p", a) != fmt.Sprintf("%p", na) {
t.Fatalf("received unexpected new buf %p != %p", a, na)
@ -28,22 +31,22 @@ func TestBufpool(t *testing.T) {
}
}
if _, err := bp.Readline(bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) {
if _, err := bp.Readline(log, bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) {
t.Fatalf("expected ErrLineTooLong, got error %v", err)
}
if _, err := bp.Readline(bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) {
if _, err := bp.Readline(log, bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) {
t.Fatalf("expected ErrLineTooLong, got error %v", err)
}
er := errReader{fmt.Errorf("bad")}
if _, err := bp.Readline(bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) {
if _, err := bp.Readline(log, bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) {
t.Fatalf("got unexpected error %s", err)
}
if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" {
if line, err := bp.Readline(log, bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" {
t.Fatalf(`got %q, err %v, expected line "ok"`, line, err)
}
if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" {
if line, err := bp.Readline(log, bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" {
t.Fatalf(`got %q, err %v, expected line "ok"`, line, err)
}
}

8
moxio/linkcopy.go
View file

@ -5,6 +5,8 @@ import (
"io"
"os"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog"
)
@ -14,7 +16,7 @@ import (
// ensure the file is written on disk. Callers should also sync the directory of
// the destination file, but may want to do that after linking/copying multiple
// files. If dst was created and an error occurred, it is removed.
func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync bool) (rerr error) {
func LinkOrCopy(log mlog.Log, dst, src string, srcReaderOpt io.Reader, sync bool) (rerr error) {
// Try hardlink first.
err := os.Link(src, dst)
if err == nil {
@ -48,7 +50,7 @@ func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync boo
err := df.Close()
log.Check(err, "closing partial destination file")
err = os.Remove(dst)
log.Check(err, "removing partial destination file", mlog.Field("path", dst))
log.Check(err, "removing partial destination file", slog.String("path", dst))
}
}()
@ -64,7 +66,7 @@ func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync boo
df = nil
if err != nil {
err := os.Remove(dst)
log.Check(err, "removing partial destination file", mlog.Field("path", dst))
log.Check(err, "removing partial destination file", slog.String("path", dst))
return err
}
return nil

2
moxio/linkcopy_test.go
View file

@ -17,7 +17,7 @@ func tcheckf(t *testing.T, err error, format string, args ...any) {
}
func TestLinkOrCopy(t *testing.T) {
log := mlog.New("linkorcopy")
log := mlog.New("linkorcopy", nil)
// link in same directory. file exists error. link to file in non-existent
// directory (exists error). link to file in system temp dir (hopefully other file

6
moxio/syncdir.go
View file

@ -5,16 +5,18 @@ package moxio
import (
"fmt"
"os"
"github.com/mjl-/mox/mlog"
)
// SyncDir opens a directory and syncs its contents to disk.
func SyncDir(dir string) error {
func SyncDir(log mlog.Log, dir string) error {
d, err := os.Open(dir)
if err != nil {
return fmt.Errorf("open directory: %v", err)
}
err = d.Sync()
xerr := d.Close()
xlog.Check(xerr, "closing directory after sync")
log.Check(xerr, "closing directory after sync")
return err
}

22
moxio/trace.go
View file

@ -3,43 +3,45 @@ package moxio
import (
"io"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog"
)
type TraceWriter struct {
log *mlog.Log
log mlog.Log
prefix string
w io.Writer
level mlog.Level
level slog.Level
}
// NewTraceWriter wraps "w" into a writer that logs all writes to "log" with
// log level trace, prefixed with "prefix".
func NewTraceWriter(log *mlog.Log, prefix string, w io.Writer) *TraceWriter {
func NewTraceWriter(log mlog.Log, prefix string, w io.Writer) *TraceWriter {
return &TraceWriter{log, prefix, w, mlog.LevelTrace}
}
// Write logs a trace line for writing buf to the client, then writes to the
// client.
func (w *TraceWriter) Write(buf []byte) (int, error) {
w.log.Trace(w.level, w.prefix+string(buf))
w.log.Trace(w.level, w.prefix, buf)
return w.w.Write(buf)
}
func (w *TraceWriter) SetTrace(level mlog.Level) {
func (w *TraceWriter) SetTrace(level slog.Level) {
w.level = level
}
type TraceReader struct {
log *mlog.Log
log mlog.Log
prefix string
r io.Reader
level mlog.Level
level slog.Level
}
// NewTraceReader wraps reader "r" into a reader that logs all reads to "log"
// with log level trace, prefixed with "prefix".
func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) *TraceReader {
func NewTraceReader(log mlog.Log, prefix string, r io.Reader) *TraceReader {
return &TraceReader{log, prefix, r, mlog.LevelTrace}
}
@ -48,11 +50,11 @@ func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) *TraceReader {
func (r *TraceReader) Read(buf []byte) (int, error) {
n, err := r.r.Read(buf)
if n > 0 {
r.log.Trace(r.level, r.prefix+string(buf[:n]))
r.log.Trace(r.level, r.prefix, buf[:n])
}
return n, err
}
func (r *TraceReader) SetTrace(level mlog.Level) {
func (r *TraceReader) SetTrace(level slog.Level) {
r.level = level
}

28
mtasts/mtasts.go
View file

@ -20,6 +20,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -31,8 +33,6 @@ import (
"github.com/mjl-/mox/moxio"
)
var xlog = mlog.New("mtasts")
var (
metricGet = promauto.NewHistogramVec(
prometheus.HistogramOpts{
@ -190,11 +190,11 @@ var (
// LookupRecord looks up the MTA-STS TXT DNS record at "_mta-sts.<domain>",
// following CNAME records, and returns the parsed MTA-STS record and the DNS TXT
// record.
func LookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) {
log := xlog.WithContext(ctx)
func LookupRecord(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) {
log := mlog.New("mtasts", elog)
start := time.Now()
defer func() {
log.Debugx("mtasts lookup result", rerr, mlog.Field("domain", domain), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start)))
log.Debugx("mtasts lookup result", rerr, slog.Any("domain", domain), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start)))
}()
// ../rfc/8461:289
@ -261,11 +261,11 @@ var HTTPClient = &http.Client{
//
// If an error is returned, callers should back off for 5 minutes until the next
// attempt.
func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policyText string, rerr error) {
log := xlog.WithContext(ctx)
func FetchPolicy(ctx context.Context, elog *slog.Logger, domain dns.Domain) (policy *Policy, policyText string, rerr error) {
log := mlog.New("mtasts", elog)
start := time.Now()
defer func() {
log.Debugx("mtasts fetch policy result", rerr, mlog.Field("domain", domain), mlog.Field("policy", policy), mlog.Field("policytext", policyText), mlog.Field("duration", time.Since(start)))
log.Debugx("mtasts fetch policy result", rerr, slog.Any("domain", domain), slog.Any("policy", policy), slog.String("policytext", policyText), slog.Duration("duration", time.Since(start)))
}()
// Timeout of 1 minute. ../rfc/8461:569
@ -291,7 +291,7 @@ func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policy
// We pass along underlying TLS certificate errors.
return nil, "", fmt.Errorf("%w: http get: %w", ErrPolicyFetch, err)
}
metrics.HTTPClientObserve(ctx, "mtasts", req.Method, resp.StatusCode, err, start)
metrics.HTTPClientObserve(ctx, log, "mtasts", req.Method, resp.StatusCode, err, start)
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, "", ErrNoPolicy
@ -329,22 +329,22 @@ func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policy
// record is still returned.
//
// Also see Get in package mtastsdb.
func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (record *Record, policy *Policy, policyText string, err error) {
log := xlog.WithContext(ctx)
func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (record *Record, policy *Policy, policyText string, err error) {
log := mlog.New("mtasts", elog)
start := time.Now()
result := "lookuperror"
defer func() {
metricGet.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("mtasts get result", err, mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("policy", policy), mlog.Field("duration", time.Since(start)))
log.Debugx("mtasts get result", err, slog.Any("domain", domain), slog.Any("record", record), slog.Any("policy", policy), slog.Duration("duration", time.Since(start)))
}()
record, _, err = LookupRecord(ctx, resolver, domain)
record, _, err = LookupRecord(ctx, log.Logger, resolver, domain)
if err != nil {
return nil, nil, "", err
}
result = "fetcherror"
policy, policyText, err = FetchPolicy(ctx, domain)
policy, policyText, err = FetchPolicy(ctx, log.Logger, domain)
if err != nil {
return record, nil, "", err
}

17
mtasts/mtasts_test.go
View file

@ -8,7 +8,7 @@ import (
"crypto/x509"
"errors"
"io"
"log"
golog "log"
"math/big"
"net"
"net/http"
@ -18,6 +18,8 @@ import (
"testing"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/adns"
"github.com/mjl-/mox/dns"
@ -25,7 +27,8 @@ import (
)
func TestLookup(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug})
mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug})
log := mlog.New("mtasts", nil)
resolver := dns.MockResolver{
TXT: map[string][]string{
@ -50,7 +53,7 @@ func TestLookup(t *testing.T) {
test := func(host string, expRecord *Record, expErr error) {
t.Helper()
record, _, err := LookupRecord(context.Background(), resolver, dns.Domain{ASCII: host})
record, _, err := LookupRecord(context.Background(), log.Logger, resolver, dns.Domain{ASCII: host})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("lookup: got err %#v, expected %#v", err, expErr)
}
@ -184,6 +187,8 @@ func fakeCert(t *testing.T, expired bool) tls.Certificate {
}
func TestFetch(t *testing.T) {
log := mlog.New("mtasts", nil)
certok := fakeCert(t, false)
certbad := fakeCert(t, true)
@ -218,7 +223,7 @@ func TestFetch(t *testing.T) {
TLSConfig: &tls.Config{
Certificates: []tls.Certificate{cert},
},
ErrorLog: log.New(io.Discard, "", 0),
ErrorLog: golog.New(io.Discard, "", 0),
}
s.ServeTLS(l, "", "")
}()
@ -235,7 +240,7 @@ func TestFetch(t *testing.T) {
},
}
p, _, err := FetchPolicy(context.Background(), dns.Domain{ASCII: domain})
p, _, err := FetchPolicy(context.Background(), log.Logger, dns.Domain{ASCII: domain})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("policy: got err %#v, expected %#v", err, expErr)
}
@ -247,7 +252,7 @@ func TestFetch(t *testing.T) {
expErr = ErrNoRecord
}
_, p, _, err = Get(context.Background(), resolver, dns.Domain{ASCII: domain})
_, p, _, err = Get(context.Background(), log.Logger, resolver, dns.Domain{ASCII: domain})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("get: got err %#v, expected %#v", err, expErr)
}

23
mtastsdb/db.go
View file

@ -16,6 +16,8 @@ import (
"sync"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -28,8 +30,6 @@ import (
"github.com/mjl-/mox/tlsrpt"
)
var xlog = mlog.New("mtastsdb")
var (
metricGet = promauto.NewCounterVec(
prometheus.CounterOpts{
@ -108,7 +108,7 @@ func Close() {
defer mutex.Unlock()
if DB != nil {
err := DB.Close()
xlog.Check(err, "closing database")
mlog.New("mtastsdb", nil).Check(err, "closing database")
DB = nil
}
}
@ -119,8 +119,7 @@ func Close() {
//
// Returns ErrNotFound if record is not present.
// Returns ErrBackoff if a recent attempt to fetch a record failed.
func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) {
log := xlog.WithContext(ctx)
func lookup(ctx context.Context, log mlog.Log, domain dns.Domain) (*PolicyRecord, error) {
db, err := database(ctx)
if err != nil {
return nil, err
@ -222,8 +221,8 @@ func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) {
// Get returns an "sts" or "no-policy-found" in reportResult in most cases (when
// not a local/internal error). It may add an "sts" result without policy contents
// ("policy-string") in case of errors while fetching the policy.
func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) {
log := xlog.WithContext(ctx)
func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) {
log := mlog.New("mtastsdb", elog)
defer func() {
result := "ok"
if err != nil && errors.Is(err, ErrBackoff) {
@ -234,16 +233,16 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
result = "error"
}
metricGet.WithLabelValues(result).Inc()
log.Debugx("mtastsdb get result", err, mlog.Field("domain", domain), mlog.Field("fresh", fresh))
log.Debugx("mtastsdb get result", err, slog.Any("domain", domain), slog.Bool("fresh", fresh))
}()
cachedPolicy, err := lookup(ctx, domain)
cachedPolicy, err := lookup(ctx, log, domain)
if err != nil && errors.Is(err, ErrNotFound) {
// We don't have a policy for this domain, not even a record that we tried recently
// and should backoff. So attempt to fetch policy.
nctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
record, p, ptext, err := mtasts.Get(nctx, resolver, domain)
record, p, ptext, err := mtasts.Get(nctx, log.Logger, resolver, domain)
if err != nil {
switch {
case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax):
@ -303,7 +302,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
policy = &cachedPolicy.Policy
nctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
record, _, err := mtasts.LookupRecord(nctx, resolver, domain)
record, _, err := mtasts.LookupRecord(nctx, log.Logger, resolver, domain)
if err != nil {
if errors.Is(err, mtasts.ErrNoRecord) {
if policy.Mode != mtasts.ModeNone {
@ -336,7 +335,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
// didn't store the raw policy lines in the past.
nctx, cancel = context.WithTimeout(ctx, 30*time.Second)
defer cancel()
p, ptext, err := mtasts.FetchPolicy(nctx, domain)
p, ptext, err := mtasts.FetchPolicy(nctx, log.Logger, domain)
if err != nil {
log.Errorx("fetching updated policy for domain, continuing with previously cached policy", err)

13
mtastsdb/db_test.go
View file

@ -12,6 +12,7 @@ import (
"time"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/mtasts"
)
@ -32,6 +33,8 @@ func TestDB(t *testing.T) {
os.Remove(dbpath)
defer os.Remove(dbpath)
log := mlog.New("mtastsdb", nil)
if err := Init(false); err != nil {
t.Fatalf("init database: %s", err)
}
@ -42,7 +45,7 @@ func TestDB(t *testing.T) {
timeNow = func() time.Time { return now }
defer func() { timeNow = time.Now }()
if p, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != ErrNotFound {
if p, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != ErrNotFound {
t.Fatalf("expected not found, got %v, %#v", err, p)
}
@ -59,7 +62,7 @@ func TestDB(t *testing.T) {
if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "123", &policy1, policy1.String()); err != nil {
t.Fatalf("upsert record: %s", err)
}
if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil {
if got, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != nil {
t.Fatalf("lookup after insert: %s", err)
} else if !reflect.DeepEqual(got.Policy, policy1) {
t.Fatalf("mismatch between inserted and retrieved: got %#v, want %#v", got, policy1)
@ -76,7 +79,7 @@ func TestDB(t *testing.T) {
if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "124", &policy2, policy2.String()); err != nil {
t.Fatalf("upsert record: %s", err)
}
if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil {
if got, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != nil {
t.Fatalf("lookup after insert: %s", err)
} else if !reflect.DeepEqual(got.Policy, policy2) {
t.Fatalf("mismatch between inserted and retrieved: got %v, want %v", got, policy2)
@ -108,7 +111,7 @@ func TestDB(t *testing.T) {
t.Fatalf("records mismatch, got %#v, expected %#v", records, expRecords)
}
if _, err := lookup(ctxbg, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff {
if _, err := lookup(ctxbg, log, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff {
t.Fatalf("got %#v, expected ErrBackoff", err)
}
@ -125,7 +128,7 @@ func TestDB(t *testing.T) {
testGet := func(domain string, expPolicy *mtasts.Policy, expFresh bool, expErr error) {
t.Helper()
p, _, fresh, err := Get(ctxbg, resolver, dns.Domain{ASCII: domain})
p, _, fresh, err := Get(ctxbg, log.Logger, resolver, dns.Domain{ASCII: domain})
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %v, expected %v", err, expErr)
}

37
mtastsdb/refresh.go
View file

@ -8,6 +8,8 @@ import (
"runtime/debug"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
@ -28,11 +30,9 @@ func refresh() int {
for {
ticker.Reset(interval)
ctx := context.WithValue(mox.Context, mlog.CidKey, mox.Cid())
n, err := refresh1(ctx, dns.StrictResolver{Pkg: "mtastsdb"}, time.Sleep)
if err != nil {
xlog.WithContext(ctx).Errorx("periodic refresh of cached mtasts policies", err)
}
log := mlog.New("mtastsdb", nil).WithCid(mox.Cid())
n, err := refresh1(mox.Context, log, dns.StrictResolver{Pkg: "mtastsdb"}, time.Sleep)
log.Check(err, "periodic refresh of cached mtasts policies")
if n > 0 {
refreshed += n
}
@ -51,7 +51,7 @@ func refresh() int {
// refreshes evenly over the next 3 hours, randomizing the domains, and we add some
// jitter to the timing. Each refresh is done in a new goroutine, so a single slow
// refresh doesn't mess up the timing.
func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) {
func refresh1(ctx context.Context, log mlog.Log, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) {
db, err := database(ctx)
if err != nil {
return 0, err
@ -87,10 +87,10 @@ func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Dura
}
// Launch goroutine with the refresh.
xlog.WithContext(ctx).Debug("will refresh mta-sts policies over next 3 hours", mlog.Field("count", len(prs)))
log.Debug("will refresh mta-sts policies over next 3 hours", slog.Int("count", len(prs)))
start := timeNow()
for i, pr := range prs {
go refreshDomain(ctx, db, resolver, pr)
go refreshDomain(ctx, log, db, resolver, pr)
if i < len(prs)-1 {
interval := 3 * int64(time.Hour) / int64(len(prs)-1)
extra := time.Duration(rand.Int63n(interval) - interval/2)
@ -104,13 +104,12 @@ func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Dura
return len(prs), nil
}
func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr PolicyRecord) {
log := xlog.WithContext(ctx)
func refreshDomain(ctx context.Context, log mlog.Log, db *bstore.DB, resolver dns.Resolver, pr PolicyRecord) {
defer func() {
x := recover()
if x != nil {
// Should not happen, but make sure errors don't take down the application.
log.Error("refresh1", mlog.Field("panic", x))
log.Error("refresh1", slog.Any("panic", x))
debug.PrintStack()
metrics.PanicInc(metrics.Mtastsdb)
}
@ -121,11 +120,11 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
d, err := dns.ParseDomain(pr.Domain)
if err != nil {
log.Errorx("refreshing mta-sts policy: parsing policy domain", err, mlog.Field("domain", d))
log.Errorx("refreshing mta-sts policy: parsing policy domain", err, slog.Any("domain", d))
return
}
log.Debug("refreshing mta-sts policy for domain", mlog.Field("domain", d))
record, _, err := mtasts.LookupRecord(ctx, resolver, d)
log.Debug("refreshing mta-sts policy for domain", slog.Any("domain", d))
record, _, err := mtasts.LookupRecord(ctx, log.Logger, resolver, d)
if err == nil && record.ID == pr.RecordID {
qup := bstore.QueryDB[PolicyRecord](ctx, db)
qup.FilterNonzero(PolicyRecord{Domain: pr.Domain, LastUpdate: pr.LastUpdate})
@ -137,7 +136,7 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
if n, err := qup.UpdateNonzero(update); err != nil {
log.Errorx("updating refreshed, unmodified policy in database", err)
} else if n != 1 {
log.Info("expected to update 1 policy after refresh", mlog.Field("count", n))
log.Info("expected to update 1 policy after refresh", slog.Int("count", n))
}
return
}
@ -152,14 +151,14 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
// ../rfc/8461:587
return
} else if err != nil {
log.Errorx("looking up mta-sts record for domain", err, mlog.Field("domain", d))
log.Errorx("looking up mta-sts record for domain", err, slog.Any("domain", d))
// Try to fetch new policy. It could be just DNS that is down. We don't want to let our policy expire.
}
p, _, err := mtasts.FetchPolicy(ctx, d)
p, _, err := mtasts.FetchPolicy(ctx, log.Logger, d)
if err != nil {
if !errors.Is(err, mtasts.ErrNoPolicy) || pr.Mode != mtasts.ModeNone {
log.Errorx("refreshing mtasts policy for domain", err, mlog.Field("domain", d))
log.Errorx("refreshing mtasts policy for domain", err, slog.Any("domain", d))
}
return
}
@ -178,6 +177,6 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
if n, err := qup.UpdateFields(update); err != nil {
log.Errorx("updating refreshed, modified policy in database", err)
} else if n != 1 {
log.Info("updating refreshed, did not update 1 policy", mlog.Field("count", n))
log.Info("updating refreshed, did not update 1 policy", slog.Int("count", n))
}
}

4
mtastsdb/refresh_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/mtasts"
)
@ -135,7 +136,8 @@ func TestRefresh(t *testing.T) {
t.Fatalf("bad sleep duration %v", d)
}
}
if n, err := refresh1(ctxbg, resolver, sleep); err != nil || n != 3 {
log := mlog.New("mtastsdb", nil)
if n, err := refresh1(ctxbg, log, resolver, sleep); err != nil || n != 3 {
t.Fatalf("refresh1: err %s, n %d, expected no error, 3", err, n)
}
if slept != 2 {

30
publicsuffix/list.go
View file

@ -18,14 +18,13 @@ import (
_ "embed"
"golang.org/x/exp/slog"
"golang.org/x/net/idna"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
)
var xlog = mlog.New("publicsuffix")
// todo: automatically fetch new lists periodically? compare it with the old one. refuse it if it changed too much, especially if it contains far fewer entries than before.
// Labels map from utf8 labels to labels for subdomains.
@ -43,16 +42,19 @@ var publicsuffixList List
var publicsuffixData []byte
func init() {
l, err := ParseList(bytes.NewReader(publicsuffixData))
log := mlog.New("publicsuffix", nil)
l, err := ParseList(log.Logger, bytes.NewReader(publicsuffixData))
if err != nil {
xlog.Fatalx("parsing public suffix list", err)
log.Fatalx("parsing public suffix list", err)
}
publicsuffixList = l
}
// ParseList parses a public suffix list.
// Only the "ICANN DOMAINS" are used.
func ParseList(r io.Reader) (List, error) {
func ParseList(elog *slog.Logger, r io.Reader) (List, error) {
log := mlog.New("publicsuffix", elog)
list := List{labels{}, labels{}}
br := bufio.NewReader(r)
@ -79,7 +81,7 @@ func ParseList(r io.Reader) (List, error) {
l = list.excludes
t = strings.Split(line, ".")
if len(t) == 1 {
xlog.Print("exclude rule with single label, skipping", mlog.Field("line", oline))
log.Print("exclude rule with single label, skipping", slog.String("line", oline))
continue
}
} else {
@ -88,19 +90,19 @@ func ParseList(r io.Reader) (List, error) {
for i := len(t) - 1; i >= 0; i-- {
w := t[i]
if w == "" {
xlog.Print("empty label in rule, skipping", mlog.Field("line", oline))
log.Print("empty label in rule, skipping", slog.String("line", oline))
break
}
if w != "" && w != "*" {
w, err = idna.Lookup.ToUnicode(w)
if err != nil {
xlog.Printx("invalid label, skipping", err, mlog.Field("line", oline))
log.Printx("invalid label, skipping", err, slog.String("line", oline))
}
}
m, ok := l[w]
if ok {
if _, dup := m[""]; i == 0 && dup {
xlog.Print("duplicate rule", mlog.Field("line", oline))
log.Print("duplicate rule", slog.String("line", oline))
}
l = m
} else {
@ -123,16 +125,16 @@ func ParseList(r io.Reader) (List, error) {
// Lookup calls Lookup on the builtin public suffix list, from
// https://publicsuffix.org/list/.
func Lookup(ctx context.Context, domain dns.Domain) (orgDomain dns.Domain) {
return publicsuffixList.Lookup(ctx, domain)
func Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) {
return publicsuffixList.Lookup(ctx, elog, domain)
}
// Lookup returns the organizational domain. If domain is an organizational
// domain, or higher-level, the same domain is returned.
func (l List) Lookup(ctx context.Context, domain dns.Domain) (orgDomain dns.Domain) {
log := xlog.WithContext(ctx)
func (l List) Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) {
log := mlog.New("publicsuffix", elog)
defer func() {
log.Debug("publicsuffix lookup result", mlog.Field("reqdom", domain), mlog.Field("orgdom", orgDomain))
log.Debug("publicsuffix lookup result", slog.Any("reqdom", domain), slog.Any("orgdom", orgDomain))
}()
t := strings.Split(domain.Name(), ".")

10
publicsuffix/list_test.go
View file

@ -7,6 +7,7 @@ import (
"testing"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
)
func TestList(t *testing.T) {
@ -27,7 +28,10 @@ bücher.example.com
ignored.example.com
`
l, err := ParseList(strings.NewReader(data))
log := mlog.New("publicsuffix", nil)
l, err := ParseList(log.Logger, strings.NewReader(data))
if err != nil {
t.Fatalf("parsing list: %s", err)
}
@ -44,7 +48,7 @@ ignored.example.com
t.Fatalf("idna to unicode org domain %q: %s", orgDomain, err)
}
r := l.Lookup(context.Background(), d)
r := l.Lookup(context.Background(), log.Logger, d)
if r != od {
t.Fatalf("got %q, expected %q, for domain %q", r, orgDomain, domain)
}
@ -70,7 +74,7 @@ ignored.example.com
test("bar.foo.xn--bcher-kva.example.com", "foo.bücher.example.com")
test("x.ignored.example.com", "example.com")
l, err = ParseList(bytes.NewReader(publicsuffixData))
l, err = ParseList(log.Logger, bytes.NewReader(publicsuffixData))
if err != nil {
t.Fatalf("parsing public suffix list: %s", err)
}

93
queue/direct.go
View file

@ -11,6 +11,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -88,7 +90,7 @@ var (
)
// todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time?
func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
func fail(qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
// todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503
// todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls?
// todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379
@ -106,18 +108,18 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
qup := bstore.QueryDB[Msg](context.Background(), DB)
qup.FilterID(m.ID)
if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil {
qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg))
qlog.Errorx("storing delivery error", err, slog.String("deliveryerror", errmsg))
}
if m.Attempts == 5 {
// We've attempted deliveries at these intervals: 0, 7.5m, 15m, 30m, 1h, 2u.
// Let sender know delivery is delayed.
qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), mlog.Field("backoff", backoff))
qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), slog.Duration("backoff", backoff))
retryUntil := m.LastAttempt.Add((4 + 8 + 16) * time.Hour)
deliverDSNDelay(qlog, m, remoteMTA, secodeOpt, errmsg, retryUntil)
} else {
qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), mlog.Field("backoff", backoff), mlog.Field("nextattempt", m.NextAttempt))
qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), slog.Duration("backoff", backoff), slog.Time("nextattempt", m.NextAttempt))
}
}
@ -129,7 +131,7 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
// domain (MTA-STS), its policy type can be empty, in which case there is no
// information (e.g. internal failure). hostResults are per-host details (DANE, one
// per MX target).
func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) (recipientDomainResult tlsrpt.Result, hostResults []tlsrpt.Result) {
func deliverDirect(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) (recipientDomainResult tlsrpt.Result, hostResults []tlsrpt.Result) {
// High-level approach:
// - Resolve domain to deliver to (CNAME), and determine hosts to try to deliver to (MX)
// - Get MTA-STS policy for domain (optional). If present, only deliver to its
@ -151,8 +153,8 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
// possibly a chain. If there are no MX records, it can be an IP or the host
// directly.
origNextHop := m.RecipientDomain.Domain
ctx := context.WithValue(mox.Context, mlog.CidKey, cid)
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog, resolver, m.RecipientDomain)
ctx := mox.Shutdown
haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog.Logger, resolver, m.RecipientDomain)
if err != nil {
// If this is a DNSSEC authentication error, we'll collect it for TLS reporting.
// Hopefully it's a temporary misconfiguration that is solve before we try to send
@ -179,14 +181,13 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
// would only take a single CNAME DNS response to direct us to an unrelated domain.
var policy *mtasts.Policy // Policy can have mode enforce, testing and none.
if !origNextHop.IsZero() {
cidctx := context.WithValue(mox.Shutdown, mlog.CidKey, cid)
policy, recipientDomainResult, _, err = mtastsdb.Get(cidctx, resolver, origNextHop)
policy, recipientDomainResult, _, err = mtastsdb.Get(ctx, qlog.Logger, resolver, origNextHop)
if err != nil {
if tlsRequiredNo {
qlog.Infox("mtasts lookup temporary error, continuing due to tls-required-no message header", err, mlog.Field("domain", origNextHop))
qlog.Infox("mtasts lookup temporary error, continuing due to tls-required-no message header", err, slog.Any("domain", origNextHop))
metricTLSRequiredNoIgnored.WithLabelValues("mtastspolicy").Inc()
} else {
qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, mlog.Field("domain", origNextHop))
qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, slog.Any("domain", origNextHop))
recipientDomainResult.Summary.TotalFailureSessionCount++
fail(qlog, m, backoff, false, dsn.NameIP{}, "", err.Error())
return
@ -226,22 +227,21 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
}
if policy.Mode == mtasts.ModeEnforce {
if tlsRequiredNo {
qlog.Info("mx host does not match mta-sts policy in mode enforce, ignoring due to tls-required-no message header", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts))
qlog.Info("mx host does not match mta-sts policy in mode enforce, ignoring due to tls-required-no message header", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts))
metricTLSRequiredNoIgnored.WithLabelValues("mtastsmx").Inc()
} else {
errmsg = fmt.Sprintf("mx host %s does not match enforced mta-sts policy with hosts %s", h.Domain, strings.Join(policyHosts, ","))
qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts))
qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts))
recipientDomainResult.Summary.TotalFailureSessionCount++
continue
}
} else {
qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts))
qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts))
}
}
qlog.Info("delivering to remote", mlog.Field("remote", h), mlog.Field("queuecid", cid))
cid := mox.Cid()
nqlog := qlog.WithCid(cid)
qlog.Info("delivering to remote", slog.Any("remote", h))
nqlog := qlog.WithCid(mox.Cid())
var remoteIP net.IP
enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce
@ -270,7 +270,7 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
var badTLS, ok bool
var hostResult tlsrpt.Result
permanent, tlsDANE, badTLS, secodeOpt, remoteIP, errmsg, hostResult, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode, tlsPKIX, &recipientDomainResult)
permanent, tlsDANE, badTLS, secodeOpt, remoteIP, errmsg, hostResult, ok = deliverHost(nqlog, resolver, dialer, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode, tlsPKIX, &recipientDomainResult)
var zerotype tlsrpt.PolicyType
if hostResult.Policy.Type != zerotype {
@ -293,8 +293,8 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
}
// todo future: add a configuration option to not fall back?
nqlog.Info("connecting again for delivery attempt without tls", mlog.Field("enforcemtasts", enforceMTASTS), mlog.Field("tlsdane", tlsDANE), mlog.Field("requiretls", m.RequireTLS))
permanent, _, _, secodeOpt, remoteIP, errmsg, _, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, smtpclient.TLSSkip, false, &tlsrpt.Result{})
nqlog.Info("connecting again for delivery attempt without tls", slog.Bool("enforcemtasts", enforceMTASTS), slog.Bool("tlsdane", tlsDANE), slog.Any("requiretls", m.RequireTLS))
permanent, _, _, secodeOpt, remoteIP, errmsg, _, ok = deliverHost(nqlog, resolver, dialer, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, smtpclient.TLSSkip, false, &tlsrpt.Result{})
}
if ok {
@ -350,7 +350,7 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
// The returned hostResult holds TLSRPT reporting results for the connection
// attempt. Its policy type can be the zero value, indicating there was no finding
// (e.g. internal error).
func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, cid int64, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode, tlsPKIX bool, recipientDomainResult *tlsrpt.Result) (permanent, tlsDANE, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, hostResult tlsrpt.Result, ok bool) {
func deliverHost(log mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode, tlsPKIX bool, recipientDomainResult *tlsrpt.Result) (permanent, tlsDANE, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, hostResult tlsrpt.Result, ok bool) {
// About attempting delivery to multiple addresses of a host: ../rfc/5321:3898
tlsRequiredNo := m.RequireTLS != nil && !*m.RequireTLS
@ -367,18 +367,18 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
}
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, mode, deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debug("queue deliverhost result",
mlog.Field("host", host),
mlog.Field("attempt", m.Attempts),
mlog.Field("tlsmode", tlsMode),
mlog.Field("tlspkix", tlsPKIX),
mlog.Field("tlsdane", tlsDANE),
mlog.Field("tlsrequiredno", tlsRequiredNo),
mlog.Field("permanent", permanent),
mlog.Field("badtls", badTLS),
mlog.Field("secodeopt", secodeOpt),
mlog.Field("errmsg", errmsg),
mlog.Field("ok", ok),
mlog.Field("duration", time.Since(start)))
slog.Any("host", host),
slog.Int("attempt", m.Attempts),
slog.Any("tlsmode", tlsMode),
slog.Bool("tlspkix", tlsPKIX),
slog.Bool("tlsdane", tlsDANE),
slog.Bool("tlsrequiredno", tlsRequiredNo),
slog.Bool("permanent", permanent),
slog.Bool("badtls", badTLS),
slog.String("secodeopt", secodeOpt),
slog.String("errmsg", errmsg),
slog.Bool("ok", ok),
slog.Duration("duration", time.Since(start)))
}()
// Open message to deliver.
@ -392,8 +392,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
log.Check(err, "closing message after delivery attempt")
}()
cidctx := context.WithValue(mox.Context, mlog.CidKey, cid)
ctx, cancel := context.WithTimeout(cidctx, 30*time.Second)
ctx, cancel := context.WithTimeout(mox.Shutdown, 30*time.Second)
defer cancel()
// We must lookup the IPs for the host name before checking DANE TLSA records. And
@ -415,10 +414,10 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
}
metricDestinations.Inc()
authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log, resolver, host, m.DialedIPs)
authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log.Logger, resolver, host, m.DialedIPs)
destAuthentic := err == nil && authentic && origNextHopAuthentic && (!haveMX || expandedNextHopAuthentic) && host.IsDomain()
if !destAuthentic {
log.Debugx("not attempting verification with dane", err, mlog.Field("authentic", authentic), mlog.Field("expandedauthentic", expandedAuthentic))
log.Debugx("not attempting verification with dane", err, slog.Bool("authentic", authentic), slog.Bool("expandedauthentic", expandedAuthentic))
// Track a DNSSEC error if found.
var errCode adns.ErrorCode
@ -447,7 +446,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
// Look for TLSA records in either the expandedHost, or otherwise the original
// host. ../rfc/7672:912
var tlsaBaseDomain dns.Domain
tlsDANE, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost)
tlsDANE, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log.Logger, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost)
if tlsDANE {
metricDestinationDANERequired.Inc()
}
@ -475,7 +474,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
},
}
} else {
log.Debug("delivery with required starttls with dane verification", mlog.Field("allowedtlshostnames", tlsHostnames))
log.Debug("delivery with required starttls with dane verification", slog.Any("allowedtlshostnames", tlsHostnames))
}
// Based on CNAMEs followed and DNSSEC-secure status, we must allow up to 4 host
// names.
@ -525,7 +524,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
conn, remoteIP, err = smtpclient.Dial(ctx, log, dialer, host, ips, 25, m.DialedIPs)
conn, remoteIP, err = smtpclient.Dial(ctx, log.Logger, dialer, host, ips, 25, m.DialedIPs)
}
cancel()
@ -543,7 +542,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
}
metricConnection.WithLabelValues(result).Inc()
if err != nil {
log.Debugx("connecting to remote smtp", err, mlog.Field("host", host))
log.Debugx("connecting to remote smtp", err, slog.Any("host", host))
return false, tlsDANE, false, "", remoteIP, fmt.Sprintf("dialing smtp server: %v", err), hostResult, false
}
@ -554,8 +553,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
rcptTo := m.Recipient().XString(m.SMTPUTF8)
// todo future: get closer to timeouts specified in rfc? ../rfc/5321:3610
log = log.Fields(mlog.Field("remoteip", remoteIP))
ctx, cancel = context.WithTimeout(cidctx, 30*time.Minute)
log = log.With(slog.Any("remoteip", remoteIP))
ctx, cancel = context.WithTimeout(mox.Shutdown, 30*time.Minute)
defer cancel()
mox.Connections.Register(conn, "smtpclient", "queue")
@ -577,7 +576,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
RecipientDomainResult: recipientDomainResult,
HostResult: &hostResult,
}
sc, err := smtpclient.New(ctx, log, conn, tlsMode, tlsPKIX, ourHostname, firstHost, opts)
sc, err := smtpclient.New(ctx, log.Logger, conn, tlsMode, tlsPKIX, ourHostname, firstHost, opts)
defer func() {
if sc == nil {
conn.Close()
@ -597,7 +596,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
STARTTLS: sc.TLSEnabled(),
RequireTLS: sc.SupportsRequireTLS(),
}
if err = updateRecipientDomainTLS(ctx, m.SenderAccount, rdt); err != nil {
if err = updateRecipientDomainTLS(ctx, log, m.SenderAccount, rdt); err != nil {
err = fmt.Errorf("storing recipient domain tls status: %w", err)
}
}
@ -658,8 +657,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
}
// Update (overwite) last known starttls/requiretls support for recipient domain.
func updateRecipientDomainTLS(ctx context.Context, senderAccount string, rdt store.RecipientDomainTLS) error {
acc, err := store.OpenAccount(senderAccount)
func updateRecipientDomainTLS(ctx context.Context, log mlog.Log, senderAccount string, rdt store.RecipientDomainTLS) error {
acc, err := store.OpenAccount(log, senderAccount)
if err != nil {
return fmt.Errorf("open account: %w", err)
}

18
queue/dsn.go
View file

@ -6,6 +6,8 @@ import (
"os"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -27,7 +29,7 @@ var (
)
)
func deliverDSNFailure(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
func deliverDSNFailure(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
const subject = "mail delivery failed"
message := fmt.Sprintf(`
Delivery has failed permanently for your email to:
@ -44,7 +46,7 @@ Error during the last delivery attempt:
deliverDSN(log, m, remoteMTA, secodeOpt, errmsg, true, nil, subject, message)
}
func deliverDSNDelay(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) {
func deliverDSNDelay(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) {
// Should not happen, but doesn't hurt to prevent sending delayed delivery
// notifications for DMARC reports. We don't want to waste postmaster attention.
if m.IsDMARCReport {
@ -72,14 +74,14 @@ Error during the last delivery attempt:
// users. So we are delivering to local users. ../rfc/5321:1466
// ../rfc/5321:1494
// ../rfc/7208:490
func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) {
func deliverDSN(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) {
kind := "delayed delivery"
if permanent {
kind = "failure"
}
qlog := func(text string, err error) {
log.Errorx("queue dsn: "+text+": sender will not be informed about dsn", err, mlog.Field("sender", m.Sender().XString(m.SMTPUTF8)), mlog.Field("kind", kind))
log.Errorx("queue dsn: "+text+": sender will not be informed about dsn", err, slog.String("sender", m.Sender().XString(m.SMTPUTF8)), slog.String("kind", kind))
}
msgf, err := os.Open(m.MessagePath())
@ -156,9 +158,9 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
// senderAccount should already by postmaster, but doesn't hurt to ensure it.
senderAccount = mox.Conf.Static.Postmaster.Account
}
acc, err := store.OpenAccount(senderAccount)
acc, err := store.OpenAccount(log, senderAccount)
if err != nil {
acc, err = store.OpenAccount(mox.Conf.Static.Postmaster.Account)
acc, err = store.OpenAccount(log, mox.Conf.Static.Postmaster.Account)
if err != nil {
qlog("looking up postmaster account after sender account was not found", err)
return
@ -167,10 +169,10 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
}
defer func() {
err := acc.Close()
log.Check(err, "queue dsn: closing account", mlog.Field("sender", m.Sender().XString(m.SMTPUTF8)), mlog.Field("kind", kind))
log.Check(err, "queue dsn: closing account", slog.String("sender", m.Sender().XString(m.SMTPUTF8)), slog.String("kind", kind))
}()
msgFile, err := store.CreateMessageTemp("queue-dsn")
msgFile, err := store.CreateMessageTemp(log, "queue-dsn")
if err != nil {
qlog("creating temporary message file", err)
return

56
queue/queue.go
View file

@ -15,6 +15,7 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"golang.org/x/net/proxy"
"github.com/prometheus/client_golang/prometheus"
@ -36,8 +37,6 @@ import (
"github.com/mjl-/mox/tlsrptdb"
)
var xlog = mlog.New("queue")
var (
metricConnection = promauto.NewCounterVec(
prometheus.CounterOpts{
@ -163,7 +162,9 @@ func Init() error {
// Shutdown closes the queue database. The delivery process isn't stopped. For tests only.
func Shutdown() {
err := DB.Close()
xlog.Check(err, "closing queue db")
if err != nil {
mlog.New("queue", nil).Errorx("closing queue db", err)
}
DB = nil
}
@ -221,7 +222,7 @@ func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf
//
// Add sets derived fields like RecipientDomainStr, and fields related to queueing,
// such as Queued, NextAttempt, LastAttempt, LastError.
func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error {
// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
if qm.ID != 0 {
@ -238,7 +239,7 @@ func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
if qm.SenderAccount == "" {
return fmt.Errorf("cannot queue with localserve without local account")
}
acc, err := store.OpenAccount(qm.SenderAccount)
acc, err := store.OpenAccount(log, qm.SenderAccount)
if err != nil {
return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
}
@ -279,14 +280,14 @@ func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
defer func() {
if dst != "" {
err := os.Remove(dst)
log.Check(err, "removing destination message file for queue", mlog.Field("path", dst))
log.Check(err, "removing destination message file for queue", slog.String("path", dst))
}
}()
dstDir := filepath.Dir(dst)
os.MkdirAll(dstDir, 0770)
if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
return fmt.Errorf("linking/copying message to new file: %s", err)
} else if err := moxio.SyncDir(dstDir); err != nil {
} else if err := moxio.SyncDir(log, dstDir); err != nil {
return fmt.Errorf("sync directory: %v", err)
}
@ -359,7 +360,7 @@ func Kick(ctx context.Context, ID int64, toDomain, recipient string, transport *
// Drop removes messages from the queue that match all nonzero parameters.
// If all parameters are zero, all messages are removed.
// Returns number of messages removed.
func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
func Drop(ctx context.Context, log mlog.Log, ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](ctx, DB)
if ID > 0 {
q.FilterID(ID)
@ -381,7 +382,7 @@ func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int
for _, m := range msgs {
p := m.MessagePath()
if err := os.Remove(p); err != nil {
xlog.WithContext(ctx).Errorx("removing queue message from file system", err, mlog.Field("queuemsgid", m.ID), mlog.Field("path", p))
log.Errorx("removing queue message from file system", err, slog.Int64("queuemsgid", m.ID), slog.String("path", p))
}
}
return n, nil
@ -427,6 +428,8 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
return err
}
log := mlog.New("queue", nil)
// High-level delivery strategy advice: ../rfc/5321:3685
go func() {
// Map keys are either dns.Domain.Name()'s, or string-formatted IP addresses.
@ -449,14 +452,14 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
continue
}
launchWork(resolver, busyDomains)
timer.Reset(nextWork(mox.Shutdown, busyDomains))
launchWork(log, resolver, busyDomains)
timer.Reset(nextWork(mox.Shutdown, log, busyDomains))
}
}()
return nil
}
func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration {
func nextWork(ctx context.Context, log mlog.Log, busyDomains map[string]struct{}) time.Duration {
q := bstore.QueryDB[Msg](ctx, DB)
if len(busyDomains) > 0 {
var doms []any
@ -471,13 +474,13 @@ func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duratio
if err == bstore.ErrAbsent {
return 24 * time.Hour
} else if err != nil {
xlog.Errorx("finding time for next delivery attempt", err)
log.Errorx("finding time for next delivery attempt", err)
return 1 * time.Minute
}
return time.Until(qm.NextAttempt)
}
func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
func launchWork(log mlog.Log, resolver dns.Resolver, busyDomains map[string]struct{}) int {
q := bstore.QueryDB[Msg](mox.Shutdown, DB)
q.FilterLessEqual("NextAttempt", time.Now())
q.SortAsc("NextAttempt")
@ -491,14 +494,14 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
}
msgs, err := q.List()
if err != nil {
xlog.Errorx("querying for work in queue", err)
log.Errorx("querying for work in queue", err)
mox.Sleep(mox.Shutdown, 1*time.Second)
return -1
}
for _, m := range msgs {
busyDomains[formatIPDomain(m.RecipientDomain)] = struct{}{}
go deliver(resolver, m)
go deliver(log, resolver, m)
}
return len(msgs)
}
@ -521,16 +524,15 @@ func queueDelete(ctx context.Context, msgID int64) error {
// deliver attempts to deliver a message.
// The queue is updated, either by removing a delivered or permanently failed
// message, or updating the time for the next attempt. A DSN may be sent.
func deliver(resolver dns.Resolver, m Msg) {
cid := mox.Cid()
qlog := xlog.WithCid(cid).Fields(mlog.Field("from", m.Sender()), mlog.Field("recipient", m.Recipient()), mlog.Field("attempts", m.Attempts), mlog.Field("msgid", m.ID))
func deliver(log mlog.Log, resolver dns.Resolver, m Msg) {
qlog := log.WithCid(mox.Cid()).With(slog.Any("from", m.Sender()), slog.Any("recipient", m.Recipient()), slog.Int("attempts", m.Attempts), slog.Int64("msgid", m.ID))
defer func() {
deliveryResult <- formatIPDomain(m.RecipientDomain)
x := recover()
if x != nil {
qlog.Error("deliver panic", mlog.Field("panic", x))
qlog.Error("deliver panic", slog.Any("panic", x))
debug.PrintStack()
metrics.PanicInc(metrics.Queue)
}
@ -578,8 +580,8 @@ func deliver(resolver dns.Resolver, m Msg) {
}
if transportName != "" {
qlog = qlog.Fields(mlog.Field("transport", transportName))
qlog.Debug("delivering with transport", mlog.Field("transport", transportName))
qlog = qlog.With(slog.String("transport", transportName))
qlog.Debug("delivering with transport")
}
// We gather TLS connection successes and failures during delivery, and we store
@ -630,7 +632,7 @@ func deliver(resolver dns.Resolver, m Msg) {
// Ensure we store policy domain in unicode in database.
policyDomain, err := dns.ParseDomain(r.Policy.Domain)
if err != nil {
qlog.Errorx("parsing policy domain for tls result", err, mlog.Field("policydomain", r.Policy.Domain))
qlog.Errorx("parsing policy domain for tls result", err, slog.String("policydomain", r.Policy.Domain))
return
}
@ -667,12 +669,12 @@ func deliver(resolver dns.Resolver, m Msg) {
var dialer smtpclient.Dialer = &net.Dialer{}
if transport.Submissions != nil {
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465)
deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465)
} else if transport.Submission != nil {
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submission, false, 587)
deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.Submission, false, 587)
} else if transport.SMTP != nil {
// todo future: perhaps also gather tlsrpt results for submissions.
deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.SMTP, false, 25)
deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.SMTP, false, 25)
} else {
ourHostname := mox.Conf.Static.HostnameDomain
if transport.Socks != nil {
@ -688,7 +690,7 @@ func deliver(resolver dns.Resolver, m Msg) {
}
ourHostname = transport.Socks.Hostname
}
recipientDomainResult, hostResults = deliverDirect(cid, qlog, resolver, dialer, ourHostname, transportName, m, backoff)
recipientDomainResult, hostResults = deliverDirect(qlog, resolver, dialer, ourHostname, transportName, m, backoff)
}
}

59
queue/queue_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/smtpclient"
@ -32,6 +33,7 @@ import (
)
var ctxbg = context.Background()
var pkglog = mlog.New("queue", nil)
func tcheck(t *testing.T, err error, msg string) {
if err != nil {
@ -61,12 +63,13 @@ func setup(t *testing.T) (*store.Account, func()) {
os.RemoveAll("../testdata/queue/data/queue")
}
log := mlog.New("queue", nil)
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/queue/mox.conf")
mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl")
acc, err := store.OpenAccount(log, "mjl")
tcheck(t, err, "open account")
err = acc.SetPassword("testtest")
err = acc.SetPassword(log, "testtest")
tcheck(t, err, "set password")
switchStop := store.Switchboard()
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
@ -88,7 +91,7 @@ test email
func prepareFile(t *testing.T) *os.File {
t.Helper()
msgFile, err := store.CreateMessageTemp("queue")
msgFile, err := store.CreateMessageTemp(pkglog, "queue")
tcheck(t, err, "create temp message for delivery to queue")
_, err = msgFile.Write([]byte(testmsg))
tcheck(t, err, "write message file")
@ -115,11 +118,11 @@ func TestQueue(t *testing.T) {
var qm Msg
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg)
@ -131,7 +134,7 @@ func TestQueue(t *testing.T) {
if msg.Attempts != 0 {
t.Fatalf("msg attempts %d, expected 0", msg.Attempts)
}
n, err := Drop(ctxbg, msgs[1].ID, "", "")
n, err := Drop(ctxbg, pkglog, msgs[1].ID, "", "")
tcheck(t, err, "drop")
if n != 1 {
t.Fatalf("dropped %d, expected 1", n)
@ -140,15 +143,15 @@ func TestQueue(t *testing.T) {
t.Fatalf("dropped message not removed from file system")
}
next := nextWork(ctxbg, nil)
next := nextWork(ctxbg, pkglog, nil)
if next > 0 {
t.Fatalf("nextWork in %s, should be now", next)
}
busy := map[string]struct{}{"mox.example": {}}
if x := nextWork(ctxbg, busy); x != 24*time.Hour {
if x := nextWork(ctxbg, pkglog, busy); x != 24*time.Hour {
t.Fatalf("nextWork in %s for busy domain, should be in 24 hours", x)
}
if nn := launchWork(nil, busy); nn != 0 {
if nn := launchWork(pkglog, nil, busy); nn != 0 {
t.Fatalf("launchWork launched %d deliveries, expected 0", nn)
}
@ -171,7 +174,7 @@ func TestQueue(t *testing.T) {
smtpclient.DialHook = nil
}()
launchWork(resolver, map[string]struct{}{})
launchWork(pkglog, resolver, map[string]struct{}{})
moxCert := fakeCert(t, "mail.mox.example", false)
@ -427,7 +430,7 @@ func TestQueue(t *testing.T) {
<-deliveryResult // Deliver sends here.
}
launchWork(resolver, map[string]struct{}{})
launchWork(pkglog, resolver, map[string]struct{}{})
waitDeliver()
return wasNetDialer
}
@ -449,7 +452,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of its route.
topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}}
qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
wasNetDialer = testDeliver(fakeSubmitServer)
if !wasNetDialer {
@ -458,7 +461,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of explicitly configured transport, that uses TLS.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
transportSubmitTLS := "submittls"
n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS)
@ -507,7 +510,7 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with socks.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
transportSocks := "socks"
n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks)
@ -523,7 +526,7 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with opportunistic TLS verification.
clearTLSResults(t)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -537,7 +540,7 @@ func TestQueue(t *testing.T) {
// Test fallback to plain text with TLS handshake fails.
clearTLSResults(t)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -557,7 +560,7 @@ func TestQueue(t *testing.T) {
},
}
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -578,7 +581,7 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with verified TLS and REQUIRETLS.
yes := true
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -595,7 +598,7 @@ func TestQueue(t *testing.T) {
},
}
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -616,7 +619,7 @@ func TestQueue(t *testing.T) {
},
}
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -638,7 +641,7 @@ func TestQueue(t *testing.T) {
// Check that message is delivered with TLS-Required: No and non-matching DANE record.
no := false
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -649,7 +652,7 @@ func TestQueue(t *testing.T) {
// Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -660,7 +663,7 @@ func TestQueue(t *testing.T) {
// Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -675,7 +678,7 @@ func TestQueue(t *testing.T) {
// Add message with requiretls that fails immediately due to no verification policy for recipient domain.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
@ -690,7 +693,7 @@ func TestQueue(t *testing.T) {
// Add another message that we'll fail to deliver entirely.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg)
@ -756,7 +759,7 @@ func TestQueue(t *testing.T) {
resolver.AllAuthentic = false
resolver.TLSA = nil
}
deliver(resolver, msg)
deliver(pkglog, resolver, msg)
err = DB.Get(ctxbg, &msg)
tcheck(t, err, "get msg")
if msg.Attempts != i {
@ -779,7 +782,7 @@ func TestQueue(t *testing.T) {
// Trigger final failure.
go func() { <-deliveryResult }() // Deliver sends here.
deliver(resolver, msg)
deliver(pkglog, resolver, msg)
err = DB.Get(ctxbg, &msg)
if err != bstore.ErrAbsent {
t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err)
@ -881,7 +884,7 @@ func TestQueueStart(t *testing.T) {
defer os.Remove(mf.Name())
defer mf.Close()
qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
err = Add(ctxbg, pkglog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
checkDialed(true)

22
queue/submit.go
View file

@ -10,6 +10,8 @@ import (
"os"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dsn"
@ -25,7 +27,7 @@ import (
// deliver via another SMTP server, e.g. relaying to a smart host, possibly
// with authentication (submission).
func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) {
// todo: configurable timeouts
port := transport.Port
@ -52,7 +54,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
var success bool
defer func() {
metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, string(tlsMode), deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second))
qlog.Debug("queue deliversubmit result", mlog.Field("host", transport.DNSHost), mlog.Field("port", port), mlog.Field("attempt", m.Attempts), mlog.Field("permanent", permanent), mlog.Field("secodeopt", secodeOpt), mlog.Field("errmsg", errmsg), mlog.Field("ok", success), mlog.Field("duration", time.Since(start)))
qlog.Debug("queue deliversubmit result", slog.Any("host", transport.DNSHost), slog.Int("port", port), slog.Int("attempt", m.Attempts), slog.Bool("permanent", permanent), slog.String("secodeopt", secodeOpt), slog.String("errmsg", errmsg), slog.Bool("ok", success), slog.Duration("duration", time.Since(start)))
}()
// todo: SMTP-DANE should be used when relaying on port 25.
@ -74,13 +76,13 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
_, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs)
_, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog.Logger, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs)
var conn net.Conn
if err == nil {
if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{}
}
conn, _, err = smtpclient.Dial(dialctx, qlog, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs)
conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs)
}
addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port))
var result string
@ -100,7 +102,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
err := conn.Close()
qlog.Check(err, "closing connection")
}
qlog.Errorx("dialing for submission", err, mlog.Field("remote", addr))
qlog.Errorx("dialing for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return
@ -122,7 +124,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
auth = append(auth, sasl.NewClientSCRAMSHA256(a.Username, a.Password))
default:
// Should not happen.
qlog.Error("missing smtp authentication mechanisms implementation", mlog.Field("mechanism", mech))
qlog.Error("missing smtp authentication mechanisms implementation", slog.String("mechanism", mech))
errmsg = fmt.Sprintf("transport %s: authentication mechanisms %q not implemented", transportName, mech)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return
@ -135,14 +137,14 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
Auth: auth,
RootCAs: mox.Conf.Static.TLS.CertPool,
}
client, err := smtpclient.New(clientctx, qlog, conn, tlsMode, tlsPKIX, mox.Conf.Static.HostnameDomain, transport.DNSHost, opts)
client, err := smtpclient.New(clientctx, qlog.Logger, conn, tlsMode, tlsPKIX, mox.Conf.Static.HostnameDomain, transport.DNSHost, opts)
if err != nil {
smtperr, ok := err.(smtpclient.Error)
var remoteMTA dsn.NameIP
if ok {
remoteMTA.Name = transport.Host
}
qlog.Errorx("establishing smtp session for submission", err, mlog.Field("remote", addr))
qlog.Errorx("establishing smtp session for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err)
secodeOpt = smtperr.Secode
fail(qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg)
@ -168,7 +170,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
p := m.MessagePath()
f, err := os.Open(p)
if err != nil {
qlog.Errorx("opening message for delivery", err, mlog.Field("remote", addr), mlog.Field("path", p))
qlog.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p))
errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return
@ -209,7 +211,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
if ok {
remoteMTA.Name = transport.Host
}
qlog.Errorx("submitting email", err, mlog.Field("remote", addr))
qlog.Errorx("submitting email", err, slog.String("remote", addr))
permanent = smtperr.Permanent
secodeOpt = smtperr.Secode
errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err)

8
quickstart.go
View file

@ -535,7 +535,7 @@ messages over SMTP.
for _, zone := range zones {
for _, ip := range hostIPs {
dnsblctx, dnsblcancel := context.WithTimeout(resolveCtx, 5*time.Second)
status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(ip))
status, expl, err := dnsbl.Lookup(dnsblctx, c.log.Logger, resolver, zone, net.ParseIP(ip))
dnsblcancel()
if status == dnsbl.StatusPass {
continue
@ -813,7 +813,7 @@ and check the admin page for the needed DNS records.`)
// Verify config.
loadTLSKeyCerts := !existingWebserver
mc, errs := mox.ParseConfig(context.Background(), filepath.FromSlash("config/mox.conf"), true, loadTLSKeyCerts, false)
mc, errs := mox.ParseConfig(context.Background(), c.log, filepath.FromSlash("config/mox.conf"), true, loadTLSKeyCerts, false)
if len(errs) > 0 {
if len(errs) > 1 {
log.Printf("checking generated config, multiple errors:")
@ -834,14 +834,14 @@ and check the admin page for the needed DNS records.`)
fatalf("cannot find domain in new config")
}
acc, _, err := store.OpenEmail(args[0])
acc, _, err := store.OpenEmail(c.log, args[0])
if err != nil {
fatalf("open account: %s", err)
}
cleanupPaths = append(cleanupPaths, dataDir, filepath.Join(dataDir, "accounts"), filepath.Join(dataDir, "accounts", accountName), filepath.Join(dataDir, "accounts", accountName, "index.db"))
password := pwgen()
if err := acc.SetPassword(password); err != nil {
if err := acc.SetPassword(c.log, password); err != nil {
fatalf("setting password: %s", err)
}
if err := acc.Close(); err != nil {

3
sendmail.go
View file

@ -17,7 +17,6 @@ import (
"github.com/mjl-/sconf"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/sasl"
"github.com/mjl-/mox/smtp"
@ -300,7 +299,7 @@ binary should be setgid that group:
Auth: auth,
RootCAs: mox.Conf.Static.TLS.CertPool,
}
client, err := smtpclient.New(ctx, mlog.New("sendmail"), conn, tlsMode, tlsPKIX, ourHostname, remoteHostname, opts)
client, err := smtpclient.New(ctx, c.log.Logger, conn, tlsMode, tlsPKIX, ourHostname, remoteHostname, opts)
xsavecheckf(err, "open smtp session")
err = client.Deliver(ctx, submitconf.From, recipient, int64(len(msg)), strings.NewReader(msg), true, false, submitconf.RequireTLS == RequireTLSYes)

2
serve.go
View file

@ -19,7 +19,7 @@ import (
"github.com/mjl-/mox/tlsrptsend"
)
func shutdown(log *mlog.Log) {
func shutdown(log mlog.Log) {
// We indicate we are shutting down. Causes new connections and new SMTP commands
// to be rejected. Should stop active connections pretty quickly.
mox.ShutdownCancel()

70
serve_unix.go
View file

@ -18,6 +18,8 @@ import (
"syscall"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -32,12 +34,12 @@ import (
"github.com/mjl-/mox/updates"
)
func monitorDNSBL(log *mlog.Log) {
func monitorDNSBL(log mlog.Log) {
defer func() {
// On error, don't bring down the entire server.
x := recover()
if x != nil {
log.Error("monitordnsbl panic", mlog.Field("panic", x))
log.Error("monitordnsbl panic", slog.Any("panic", x))
debug.PrintStack()
metrics.PanicInc(metrics.Serve)
}
@ -53,7 +55,7 @@ func monitorDNSBL(log *mlog.Log) {
for _, zone := range l.SMTP.DNSBLs {
d, err := dns.ParseDomain(zone)
if err != nil {
log.Fatalx("parsing dnsbls zone", err, mlog.Field("zone", zone))
log.Fatalx("parsing dnsbls zone", err, slog.Any("zone", zone))
}
zones = append(zones, d)
}
@ -86,9 +88,9 @@ func monitorDNSBL(log *mlog.Log) {
}
for _, zone := range zones {
status, expl, err := dnsbl.Lookup(mox.Context, resolver, zone, ip)
status, expl, err := dnsbl.Lookup(mox.Context, log.Logger, resolver, zone, ip)
if err != nil {
log.Errorx("dnsbl monitor lookup", err, mlog.Field("ip", ip), mlog.Field("zone", zone), mlog.Field("expl", expl), mlog.Field("status", status))
log.Errorx("dnsbl monitor lookup", err, slog.Any("ip", ip), slog.Any("zone", zone), slog.String("expl", expl), slog.Any("status", status))
}
k := key{zone, ip.String()}
@ -145,7 +147,7 @@ Only implemented on unix systems, not Windows.
checkACMEHosts := os.Getuid() != 0
log := mlog.New("serve")
log := c.log
if os.Getuid() == 0 {
mox.MustLoadConfig(true, checkACMEHosts)
@ -159,7 +161,7 @@ Only implemented on unix systems, not Windows.
domainsconf, err := filepath.Abs(mox.ConfigDynamicPath)
log.Check(err, "finding absolute domains.conf path")
log.Print("starting as root, initializing network listeners", mlog.Field("version", moxvar.Version), mlog.Field("pid", os.Getpid()), mlog.Field("moxconf", moxconf), mlog.Field("domainsconf", domainsconf))
log.Print("starting as root, initializing network listeners", slog.String("version", moxvar.Version), slog.Any("pid", os.Getpid()), slog.String("moxconf", moxconf), slog.String("domainsconf", domainsconf))
if os.Getenv("MOX_SOCKETS") != "" {
log.Fatal("refusing to start as root with $MOX_SOCKETS set")
}
@ -185,7 +187,7 @@ Only implemented on unix systems, not Windows.
} else {
mox.RestorePassedFiles()
mox.MustLoadConfig(true, checkACMEHosts)
log.Print("starting as unprivileged user", mlog.Field("user", mox.Conf.Static.User), mlog.Field("uid", mox.Conf.Static.UID), mlog.Field("gid", mox.Conf.Static.GID), mlog.Field("pid", os.Getpid()))
log.Print("starting as unprivileged user", slog.String("user", mox.Conf.Static.User), slog.Any("uid", mox.Conf.Static.UID), slog.Any("gid", mox.Conf.Static.GID), slog.Any("pid", os.Getpid()))
}
syscall.Umask(syscall.Umask(007) | 007)
@ -200,12 +202,12 @@ Only implemented on unix systems, not Windows.
log.Fatalx("reading random recvid data", err)
}
if err := os.WriteFile(recvidpath, recvidbuf, 0660); err != nil {
log.Fatalx("writing recvidpath", err, mlog.Field("path", recvidpath))
log.Fatalx("writing recvidpath", err, slog.String("path", recvidpath))
}
err := os.Chown(recvidpath, int(mox.Conf.Static.UID), 0)
log.Check(err, "chown receveidid.key", mlog.Field("path", recvidpath), mlog.Field("uid", mox.Conf.Static.UID), mlog.Field("gid", 0))
log.Check(err, "chown receveidid.key", slog.String("path", recvidpath), slog.Any("uid", mox.Conf.Static.UID), slog.Any("gid", 0))
err = os.Chmod(recvidpath, 0640)
log.Check(err, "chmod receveidid.key to 0640", mlog.Field("path", recvidpath))
log.Check(err, "chmod receveidid.key to 0640", slog.String("path", recvidpath))
}
if err := mox.ReceivedIDInit(recvidbuf[:16], recvidbuf[16:]); err != nil {
log.Fatalx("init receivedid", err)
@ -242,7 +244,7 @@ Only implemented on unix systems, not Windows.
// mtime. But file won't exist initially.
if !mtime.IsZero() && time.Since(mtime) < 24*time.Hour {
d := 24*time.Hour - time.Since(mtime)
log.Debug("sleeping for next check for updates", mlog.Field("sleep", d))
log.Debug("sleeping for next check for updates", slog.Duration("sleep", d))
time.Sleep(d)
next = 0
}
@ -253,12 +255,12 @@ Only implemented on unix systems, not Windows.
}
}
log.Debug("checking for updates", mlog.Field("lastknown", lastknown))
log.Debug("checking for updates", slog.Any("lastknown", lastknown))
updatesctx, updatescancel := context.WithTimeout(mox.Context, time.Minute)
latest, _, changelog, err := updates.Check(updatesctx, dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain}, lastknown, changelogURL, changelogPubKey)
latest, _, changelog, err := updates.Check(updatesctx, log.Logger, dns.StrictResolver{Log: log.Logger}, dns.Domain{ASCII: changelogDomain}, lastknown, changelogURL, changelogPubKey)
updatescancel()
if err != nil {
log.Infox("checking for updates", err, mlog.Field("latest", latest))
log.Infox("checking for updates", err, slog.Any("latest", latest))
return next
}
if !latest.After(lastknown) {
@ -266,7 +268,7 @@ Only implemented on unix systems, not Windows.
return next
}
if len(changelog.Changes) == 0 {
log.Info("new version available, but changelog is empty, ignoring", mlog.Field("latest", latest))
log.Info("new version available, but changelog is empty, ignoring", slog.Any("latest", latest))
return next
}
@ -276,7 +278,7 @@ Only implemented on unix systems, not Windows.
}
cl += "----"
a, err := store.OpenAccount(mox.Conf.Static.Postmaster.Account)
a, err := store.OpenAccount(log, mox.Conf.Static.Postmaster.Account)
if err != nil {
log.Infox("open account for postmaster changelog delivery", err)
return next
@ -285,7 +287,7 @@ Only implemented on unix systems, not Windows.
err := a.Close()
log.Check(err, "closing account")
}()
f, err := store.CreateMessageTemp("changelog")
f, err := store.CreateMessageTemp(log, "changelog")
if err != nil {
log.Infox("making temporary message file for changelog delivery", err)
return next
@ -306,7 +308,7 @@ Only implemented on unix systems, not Windows.
log.Errorx("changelog delivery", err)
return next
}
log.Info("delivered changelog", mlog.Field("current", current), mlog.Field("lastknown", lastknown), mlog.Field("latest", latest))
log.Info("delivered changelog", slog.Any("current", current), slog.Any("lastknown", lastknown), slog.Any("latest", latest))
if err := mox.StoreLastKnown(latest); err != nil {
// This will be awkward, we'll keep notifying the postmaster once every 24h...
log.Infox("updating last known version", err)
@ -353,13 +355,13 @@ Only implemented on unix systems, not Windows.
now := time.Now()
for _, e := range tmps {
if fi, err := e.Info(); err != nil {
log.Errorx("stat tmp file", err, mlog.Field("filename", e.Name()))
log.Errorx("stat tmp file", err, slog.String("filename", e.Name()))
} else if now.Sub(fi.ModTime()) > 7*24*time.Hour && !fi.IsDir() {
p := filepath.Join(tmpdir, e.Name())
if err := os.Remove(p); err != nil {
log.Errorx("removing stale temporary file", err, mlog.Field("path", p))
log.Errorx("removing stale temporary file", err, slog.String("path", p))
} else {
log.Info("removed stale temporary file", mlog.Field("path", p))
log.Info("removed stale temporary file", slog.String("path", p))
}
}
}
@ -369,7 +371,7 @@ Only implemented on unix systems, not Windows.
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
sig := <-sigc
log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig))
log.Print("shutting down, waiting max 3s for existing connections", slog.Any("signal", sig))
shutdown(log)
if num, ok := sig.(syscall.Signal); ok {
os.Exit(int(num))
@ -383,7 +385,7 @@ Only implemented on unix systems, not Windows.
// We require being able to stat the basic non-optional paths. Then we'll try to
// fix up permissions. If an error occurs when fixing permissions, we log and
// continue (could not be an actual problem).
func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid uint32) (rerr error) {
func fixperms(log mlog.Log, workdir, configdir, datadir string, moxuid, moxgid uint32) (rerr error) {
type fserr struct{ Err error }
defer func() {
x := recover()
@ -483,33 +485,33 @@ func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid
for _, ch := range changes {
if ch.uid != nil {
err := os.Chown(ch.path, int(*ch.uid), int(*ch.gid))
log.Printx("chown, fixing uid/gid", err, mlog.Field("path", ch.path), mlog.Field("olduid", ch.olduid), mlog.Field("oldgid", ch.oldgid), mlog.Field("newuid", *ch.uid), mlog.Field("newgid", *ch.gid))
log.Printx("chown, fixing uid/gid", err, slog.String("path", ch.path), slog.Any("olduid", ch.olduid), slog.Any("oldgid", ch.oldgid), slog.Any("newuid", *ch.uid), slog.Any("newgid", *ch.gid))
}
if ch.mode != nil {
err := os.Chmod(ch.path, *ch.mode)
log.Printx("chmod, fixing permissions", err, mlog.Field("path", ch.path), mlog.Field("oldmode", fmt.Sprintf("%03o", ch.oldmode)), mlog.Field("newmode", fmt.Sprintf("%03o", *ch.mode)))
log.Printx("chmod, fixing permissions", err, slog.String("path", ch.path), slog.Any("oldmode", fmt.Sprintf("%03o", ch.oldmode)), slog.Any("newmode", fmt.Sprintf("%03o", *ch.mode)))
}
}
walkchange := func(dir string) {
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.Printx("walk error, continuing", err, mlog.Field("path", path))
log.Printx("walk error, continuing", err, slog.String("path", path))
return nil
}
fi, err := d.Info()
if err != nil {
log.Printx("stat during walk, continuing", err, mlog.Field("path", path))
log.Printx("stat during walk, continuing", err, slog.String("path", path))
return nil
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
log.Printx("syscall stat during walk, continuing", err, mlog.Field("path", path))
log.Printx("syscall stat during walk, continuing", err, slog.String("path", path))
return nil
}
if st.Uid != moxuid || st.Gid != root {
err := os.Chown(path, int(moxuid), root)
log.Printx("walk chown, fixing uid/gid", err, mlog.Field("path", path), mlog.Field("olduid", st.Uid), mlog.Field("oldgid", st.Gid), mlog.Field("newuid", moxuid), mlog.Field("newgid", root))
log.Printx("walk chown, fixing uid/gid", err, slog.String("path", path), slog.Any("olduid", st.Uid), slog.Any("oldgid", st.Gid), slog.Any("newuid", moxuid), slog.Any("newgid", root))
}
omode := fi.Mode() & (fs.ModeSetgid | 0777)
var nmode fs.FileMode
@ -520,21 +522,21 @@ func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid
}
if omode != nmode {
err := os.Chmod(path, nmode)
log.Printx("walk chmod, fixing permissions", err, mlog.Field("path", path), mlog.Field("oldmode", fmt.Sprintf("%03o", omode)), mlog.Field("newmode", fmt.Sprintf("%03o", nmode)))
log.Printx("walk chmod, fixing permissions", err, slog.String("path", path), slog.Any("oldmode", fmt.Sprintf("%03o", omode)), slog.Any("newmode", fmt.Sprintf("%03o", nmode)))
}
return nil
})
log.Check(err, "walking dir to fix permissions", mlog.Field("dir", dir))
log.Check(err, "walking dir to fix permissions", slog.String("dir", dir))
}
// If config or data dir needed fixing, also set uid/gid and mode and files/dirs
// inside, recursively. We don't always recurse, data probably contains many files.
if fixconfig {
log.Print("fixing permissions in config dir", mlog.Field("configdir", configdir))
log.Print("fixing permissions in config dir", slog.String("configdir", configdir))
walkchange(configdir)
}
if fixdata {
log.Print("fixing permissions in data dir", mlog.Field("configdir", configdir))
log.Print("fixing permissions in data dir", slog.String("configdir", configdir))
walkchange(datadir)
}
return nil

46
smtpclient/client.go
View file

@ -16,6 +16,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -118,7 +120,7 @@ type Client struct {
w *bufio.Writer
tr *moxio.TraceReader // Kept for changing trace levels between cmd/auth/data.
tw *moxio.TraceWriter
log *mlog.Log
log mlog.Log
lastlog time.Time // For adding delta timestamps between log lines.
cmds []string // Last or active command, for generating errors and metrics.
cmdStart time.Time // Start of command.
@ -237,7 +239,7 @@ type Opts struct {
// with opportunistic TLS without PKIX verification by default. Recipient domains
// can opt-in to PKIX verification by publishing an MTA-STS policy, or opt-in to
// DANE verification by publishing DNSSEC-protected TLSA records in DNS.
func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tlsVerifyPKIX bool, ehloHostname, remoteHostname dns.Domain, opts Opts) (*Client, error) {
func New(ctx context.Context, elog *slog.Logger, conn net.Conn, tlsMode TLSMode, tlsVerifyPKIX bool, ehloHostname, remoteHostname dns.Domain, opts Opts) (*Client, error) {
ensureResult := func(r *tlsrpt.Result) *tlsrpt.Result {
if r == nil {
return &tlsrpt.Result{}
@ -259,10 +261,10 @@ func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tls
recipientDomainResult: ensureResult(opts.RecipientDomainResult),
hostResult: ensureResult(opts.HostResult),
}
c.log = log.Fields(mlog.Field("smtpclient", "")).MoreFields(func() []mlog.Pair {
c.log = mlog.New("smtpclient", elog).WithFunc(func() []slog.Attr {
now := time.Now()
l := []mlog.Pair{
mlog.Field("delta", now.Sub(c.lastlog)),
l := []slog.Attr{
slog.Duration("delta", now.Sub(c.lastlog)),
}
c.lastlog = now
return l
@ -280,7 +282,7 @@ func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tls
c.tlsResultAdd(1, 0, nil)
c.conn = tlsconn
tlsversion, ciphersuite := mox.TLSInfo(tlsconn)
c.log.Debug("tls client handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite), mlog.Field("servername", remoteHostname))
c.log.Debug("tls client handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite), slog.Any("servername", remoteHostname))
c.tls = true
} else {
c.conn = conn
@ -329,8 +331,8 @@ func (c *Client) tlsConfig() *tls.Config {
// DANE verification.
// daneRecords can be non-nil and empty, that's intended.
if c.daneRecords != nil {
verified, record, err := dane.Verify(c.log, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames)
c.log.Debugx("dane verification", err, mlog.Field("verified", verified), mlog.Field("record", record))
verified, record, err := dane.Verify(c.log.Logger, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames)
c.log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record))
if verified {
if c.daneVerifiedRecord != nil {
*c.daneVerifiedRecord = record
@ -426,7 +428,7 @@ func (c *Client) xerrorf(permanent bool, code int, secode, lastLine, format stri
type timeoutWriter struct {
conn net.Conn
timeout time.Duration
log *mlog.Log
log mlog.Log
}
func (w timeoutWriter) Write(buf []byte) (int, error) {
@ -445,7 +447,7 @@ func (c *Client) readline() (string, error) {
c.log.Errorx("setting read deadline", err)
}
line, err := bufs.Readline(c.r)
line, err := bufs.Readline(c.log, c.r)
if err != nil {
// See if this is a TLS alert from remote, and one other than 0 (which notifies
// that the connection is being closed. If so, we register a TLS connection
@ -463,7 +465,7 @@ func (c *Client) readline() (string, error) {
return line, nil
}
func (c *Client) xtrace(level mlog.Level) func() {
func (c *Client) xtrace(level slog.Level) func() {
c.xflush()
c.tr.SetTrace(level)
c.tw.SetTrace(level)
@ -543,7 +545,7 @@ func (c *Client) readecode(ecodes bool) (code int, secode, lastLine string, text
}
}
metricCommands.WithLabelValues(cmd, fmt.Sprintf("%d", co), sec).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
c.log.Debug("smtpclient command result", mlog.Field("cmd", cmd), mlog.Field("code", co), mlog.Field("secode", sec), mlog.Field("duration", time.Since(c.cmdStart)))
c.log.Debug("smtpclient command result", slog.String("cmd", cmd), slog.Int("code", co), slog.String("secode", sec), slog.Duration("duration", time.Since(c.cmdStart)))
}
return co, sec, line, texts, nil
}
@ -726,7 +728,7 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do
// Attempt TLS if remote understands STARTTLS and we aren't doing immediate TLS or if caller requires it.
if c.extStartTLS && tlsMode == TLSOpportunistic || tlsMode == TLSRequiredStartTLS {
c.log.Debug("starting tls client", mlog.Field("tlsmode", tlsMode), mlog.Field("servername", c.remoteHostname))
c.log.Debug("starting tls client", slog.Any("tlsmode", tlsMode), slog.Any("servername", c.remoteHostname))
c.cmds[0] = "starttls"
c.cmdStart = time.Now()
c.xwritelinef("STARTTLS")
@ -772,14 +774,14 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do
tlsversion, ciphersuite := mox.TLSInfo(nconn)
c.log.Debug("starttls client handshake done",
mlog.Field("tlsmode", tlsMode),
mlog.Field("verifypkix", c.tlsVerifyPKIX),
mlog.Field("verifydane", c.daneRecords != nil),
mlog.Field("ignoretlsverifyerrors", c.ignoreTLSVerifyErrors),
mlog.Field("tls", tlsversion),
mlog.Field("ciphersuite", ciphersuite),
mlog.Field("servername", c.remoteHostname),
mlog.Field("danerecord", c.daneVerifiedRecord))
slog.Any("tlsmode", tlsMode),
slog.Bool("verifypkix", c.tlsVerifyPKIX),
slog.Bool("verifydane", c.daneRecords != nil),
slog.Bool("ignoretlsverifyerrors", c.ignoreTLSVerifyErrors),
slog.String("tls", tlsversion),
slog.String("ciphersuite", ciphersuite),
slog.Any("servername", c.remoteHostname),
slog.Any("danerecord", c.daneVerifiedRecord))
c.tls = true
// Track successful TLS connection. ../rfc/8460:515
c.tlsResultAdd(1, 0, nil)
@ -1171,7 +1173,7 @@ func (c *Client) Close() (rerr error) {
c.xwriteline("QUIT")
if err := c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
c.log.Infox("setting read deadline for reading quit response", err)
} else if _, err := bufs.Readline(c.r); err != nil {
} else if _, err := bufs.Readline(c.log, c.r); err != nil {
rerr = fmt.Errorf("reading response to quit command: %v", err)
c.log.Debugx("reading quit response", err)
}

36
smtpclient/client_test.go
View file

@ -21,6 +21,8 @@ import (
"testing"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/sasl"
@ -33,9 +35,9 @@ var localhost = dns.Domain{ASCII: "localhost"}
func TestClient(t *testing.T) {
ctx := context.Background()
log := mlog.New("smtpclient")
log := mlog.New("smtpclient", nil)
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelTrace})
mlog.SetConfig(map[string]slog.Level{"": mlog.LevelTrace})
type options struct {
pipelining bool
@ -281,7 +283,7 @@ func TestClient(t *testing.T) {
result <- err
panic("stop")
}
c, err := New(ctx, log, clientConn, opts.tlsMode, opts.tlsPKIX, localhost, opts.tlsHostname, Opts{Auth: auths, RootCAs: opts.roots})
c, err := New(ctx, log.Logger, clientConn, opts.tlsMode, opts.tlsPKIX, localhost, opts.tlsHostname, Opts{Auth: auths, RootCAs: opts.roots})
if (err == nil) != (expClientErr == nil) || err != nil && !errors.As(err, reflect.New(reflect.ValueOf(expClientErr).Type()).Interface()) && !errors.Is(err, expClientErr) {
fail("new client: got err %v, expected %#v", err, expClientErr)
}
@ -382,13 +384,13 @@ test
func TestErrors(t *testing.T) {
ctx := context.Background()
log := mlog.New("")
log := mlog.New("smtpclient", nil)
// Invalid greeting.
run(t, func(s xserver) {
s.writeline("bogus") // Invalid, should be "220 <hostname>".
}, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
_, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error
if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err))
@ -399,7 +401,7 @@ func TestErrors(t *testing.T) {
run(t, func(s xserver) {
s.conn.Close()
}, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
_, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error
if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v (%v), expected ErrUnexpectedEOF without Permanent", err, err))
@ -410,7 +412,7 @@ func TestErrors(t *testing.T) {
run(t, func(s xserver) {
s.writeline("521 not accepting connections")
}, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
_, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error
if err == nil || !errors.Is(err, ErrStatus) || !errors.As(err, &xerr) || !xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrStatus with Permanent", err))
@ -421,7 +423,7 @@ func TestErrors(t *testing.T) {
run(t, func(s xserver) {
s.writeline("2200 mox.example") // Invalid, too many digits.
}, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
_, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error
if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err))
@ -435,7 +437,7 @@ func TestErrors(t *testing.T) {
s.writeline("250-mox.example")
s.writeline("500 different code") // Invalid.
}, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
_, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
var xerr Error
if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err))
@ -451,7 +453,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:")
s.writeline("550 5.7.0 not allowed")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil {
panic(err)
}
@ -471,7 +473,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:")
s.writeline("451 bad sender")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil {
panic(err)
}
@ -493,7 +495,7 @@ func TestErrors(t *testing.T) {
s.readline("RCPT TO:")
s.writeline("451")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil {
panic(err)
}
@ -517,7 +519,7 @@ func TestErrors(t *testing.T) {
s.readline("DATA")
s.writeline("550 no!")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil {
panic(err)
}
@ -537,7 +539,7 @@ func TestErrors(t *testing.T) {
s.readline("STARTTLS")
s.writeline("502 command not implemented")
}, func(conn net.Conn) {
_, err := New(ctx, log, conn, TLSRequiredStartTLS, true, localhost, dns.Domain{ASCII: "mox.example"}, Opts{})
_, err := New(ctx, log.Logger, conn, TLSRequiredStartTLS, true, localhost, dns.Domain{ASCII: "mox.example"}, Opts{})
var xerr Error
if err == nil || !errors.Is(err, ErrTLS) || !errors.As(err, &xerr) || !xerr.Permanent {
panic(fmt.Errorf("got %#v, expected ErrTLS with Permanent", err))
@ -553,7 +555,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:")
s.writeline("451 enough")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSSkip, false, localhost, dns.Domain{ASCII: "mox.example"}, Opts{})
c, err := New(ctx, log.Logger, conn, TLSSkip, false, localhost, dns.Domain{ASCII: "mox.example"}, Opts{})
if err != nil {
panic(err)
}
@ -583,7 +585,7 @@ func TestErrors(t *testing.T) {
s.readline("DATA")
s.writeline("550 not now")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil {
panic(err)
}
@ -613,7 +615,7 @@ func TestErrors(t *testing.T) {
s.readline("MAIL FROM:")
s.writeline("550 ok")
}, func(conn net.Conn) {
c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{})
if err != nil {
panic(err)
}

11
smtpclient/dial.go
View file

@ -6,6 +6,8 @@ import (
"net"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
@ -51,7 +53,8 @@ type Dialer interface {
// If we have fully specified local SMTP listener IPs, we set those for the
// outgoing connection. The admin probably configured these same IPs in SPF, but
// others possibly not.
func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) {
func Dial(ctx context.Context, elog *slog.Logger, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) {
log := mlog.New("smtpclient", elog)
timeout := 30 * time.Second
if deadline, ok := ctx.Deadline(); ok && len(ips) > 0 {
timeout = time.Until(deadline) / time.Duration(len(ips))
@ -61,7 +64,7 @@ func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain,
var lastIP net.IP
for _, ip := range ips {
addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port))
log.Debug("dialing host", mlog.Field("addr", addr))
log.Debug("dialing host", slog.String("addr", addr))
var laddr net.Addr
for _, lip := range mox.Conf.Static.SpecifiedSMTPListenIPs {
ipIs4 := ip.To4() != nil
@ -73,12 +76,12 @@ func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain,
}
conn, err := dial(ctx, dialer, timeout, addr, laddr)
if err == nil {
log.Debug("connected to host", mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr))
log.Debug("connected to host", slog.Any("host", host), slog.String("addr", addr), slog.Any("laddr", laddr))
name := host.String()
dialedIPs[name] = append(dialedIPs[name], ip)
return conn, ip, nil
}
log.Debugx("connection attempt", err, mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr))
log.Debugx("connection attempt", err, slog.Any("host", host), slog.String("addr", addr), slog.Any("laddr", laddr))
lastErr = err
lastIP = ip
}

10
smtpclient/dial_test.go
View file

@ -14,7 +14,7 @@ import (
func TestDialHost(t *testing.T) {
// We mostly want to test that dialing a second time switches to the other address family.
ctxbg := context.Background()
log := mlog.New("smtpclient")
log := mlog.New("smtpclient", nil)
resolver := dns.MockResolver{
A: map[string][]string{
@ -37,20 +37,20 @@ func TestDialHost(t *testing.T) {
}
dialedIPs := map[string][]net.IP{}
_, _, _, ips, dualstack, err := GatherIPs(ctxbg, log, resolver, ipdomain("dualstack.example"), dialedIPs)
_, _, _, ips, dualstack, err := GatherIPs(ctxbg, log.Logger, resolver, ipdomain("dualstack.example"), dialedIPs)
if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}) || !dualstack {
t.Fatalf("expected err nil, address 10.0.0.1,2001:db8::1, dualstack true, got %v %v %v", err, ips, dualstack)
}
_, ip, err := Dial(ctxbg, log, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs)
_, ip, err := Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs)
if err != nil || ip.String() != "10.0.0.1" {
t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack)
}
_, _, _, ips, dualstack, err = GatherIPs(ctxbg, log, resolver, ipdomain("dualstack.example"), dialedIPs)
_, _, _, ips, dualstack, err = GatherIPs(ctxbg, log.Logger, resolver, ipdomain("dualstack.example"), dialedIPs)
if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("10.0.0.1")}) || !dualstack {
t.Fatalf("expected err nil, address 2001:db8::1,10.0.0.1, dualstack true, got %v %v %v", err, ips, dualstack)
}
_, ip, err = Dial(ctxbg, log, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs)
_, ip, err = Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs)
if err != nil || ip.String() != "2001:db8::1" {
t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack)
}

36
smtpclient/gather.go
View file

@ -12,6 +12,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/adns"
"github.com/mjl-/mox/dns"
@ -45,9 +47,11 @@ var (
// were found, both the original and expanded next-hops must be authentic for DANE
// to apply. For a non-IP with no MX records found, the authentic result can be
// used to decide which of the names to use as TLSA base domain.
func GatherDestinations(ctx context.Context, log *mlog.Log, resolver dns.Resolver, origNextHop dns.IPDomain) (haveMX, origNextHopAuthentic, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, hosts []dns.IPDomain, permanent bool, err error) {
func GatherDestinations(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, origNextHop dns.IPDomain) (haveMX, origNextHopAuthentic, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, hosts []dns.IPDomain, permanent bool, err error) {
// ../rfc/5321:3824
log := mlog.New("smtpclient", elog)
// IP addresses are dialed directly, and don't have TLSA records.
if len(origNextHop.IP) > 0 {
return false, false, false, expandedNextHop, []dns.IPDomain{origNextHop}, false, nil
@ -167,7 +171,9 @@ func GatherDestinations(ctx context.Context, log *mlog.Log, resolver dns.Resolve
// GatherIPs looks up the IPs to try for connecting to host, with the IPs ordered
// to take previous attempts into account. For use with DANE, the CNAME-expanded
// name is returned, and whether the DNS responses were authentic.
func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host dns.IPDomain, dialedIPs map[string][]net.IP) (authentic bool, expandedAuthentic bool, expandedHost dns.Domain, ips []net.IP, dualstack bool, rerr error) {
func GatherIPs(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, host dns.IPDomain, dialedIPs map[string][]net.IP) (authentic bool, expandedAuthentic bool, expandedHost dns.Domain, ips []net.IP, dualstack bool, rerr error) {
log := mlog.New("smtpclient", elog)
if len(host.IP) > 0 {
return false, false, dns.Domain{}, []net.IP{host.IP}, false, nil
}
@ -250,7 +256,7 @@ func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host d
// Prefer "i" if it is the same as last and we should be preferring it.
return preferPrev && ips[i].Equal(prevIP)
})
log.Debug("ordered ips for dialing", mlog.Field("ips", ips))
log.Debug("ordered ips for dialing", slog.Any("ips", ips))
}
return
}
@ -268,7 +274,9 @@ func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host d
// must do TLS, but not verify the remote TLS certificate.
//
// Returned values are always meaningful, also when an error was returned.
func GatherTLSA(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain) (daneRequired bool, daneRecords []adns.TLSA, tlsaBaseDomain dns.Domain, err error) {
func GatherTLSA(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain) (daneRequired bool, daneRecords []adns.TLSA, tlsaBaseDomain dns.Domain, err error) {
log := mlog.New("smtpclient", elog)
// ../rfc/7672:912
// This function is only called when the lookup of host was authentic.
@ -288,32 +296,32 @@ func GatherTLSA(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host
}
if len(l) == 0 || err != nil {
daneRequired = err != nil
log.Debugx("gathering tlsa records failed", err, mlog.Field("danerequired", daneRequired), mlog.Field("basedomain", tlsaBaseDomain))
log.Debugx("gathering tlsa records failed", err, slog.Bool("danerequired", daneRequired), slog.Any("basedomain", tlsaBaseDomain))
return daneRequired, nil, tlsaBaseDomain, err
}
daneRequired = len(l) > 0
l = filterUsableTLSARecords(log, l)
log.Debug("tlsa records exist", mlog.Field("danerequired", daneRequired), mlog.Field("records", l), mlog.Field("basedomain", tlsaBaseDomain))
log.Debug("tlsa records exist", slog.Bool("danerequired", daneRequired), slog.Any("records", l), slog.Any("basedomain", tlsaBaseDomain))
return daneRequired, l, tlsaBaseDomain, err
}
// lookupTLSACNAME composes a TLSA domain name to lookup, follows CNAMEs and looks
// up TLSA records. no TLSA records exist, a nil error is returned as it means
// the host does not opt-in to DANE.
func lookupTLSACNAME(ctx context.Context, log *mlog.Log, resolver dns.Resolver, port int, protocol string, host dns.Domain) (l []adns.TLSA, rerr error) {
func lookupTLSACNAME(ctx context.Context, log mlog.Log, resolver dns.Resolver, port int, protocol string, host dns.Domain) (l []adns.TLSA, rerr error) {
name := fmt.Sprintf("_%d._%s.%s", port, protocol, host.ASCII+".")
for i := 0; ; i++ {
cname, result, err := resolver.LookupCNAME(ctx, name)
if dns.IsNotFound(err) {
if !result.Authentic {
log.Debugx("cname nxdomain result during tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name))
log.Debugx("cname nxdomain result during tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name))
return nil, nil
}
break
} else if err != nil {
return nil, fmt.Errorf("looking up cname for tlsa candidate base domain: %w", err)
} else if !result.Authentic {
log.Debugx("cname result during tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name))
log.Debugx("cname result during tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name))
return nil, nil
}
if i == 10 {
@ -325,18 +333,18 @@ func lookupTLSACNAME(ctx context.Context, log *mlog.Log, resolver dns.Resolver,
var err error
l, result, err = resolver.LookupTLSA(ctx, 0, "", name)
if dns.IsNotFound(err) || err == nil && len(l) == 0 {
log.Debugx("no tlsa records for host, not doing dane", err, mlog.Field("host", host), mlog.Field("name", name), mlog.Field("authentic", result.Authentic))
log.Debugx("no tlsa records for host, not doing dane", err, slog.Any("host", host), slog.String("name", name), slog.Bool("authentic", result.Authentic))
return nil, nil
} else if err != nil {
return nil, fmt.Errorf("looking up tlsa records for tlsa candidate base domain: %w", err)
} else if !result.Authentic {
log.Debugx("tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name))
log.Debugx("tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name))
return nil, nil
}
return l, nil
}
func filterUsableTLSARecords(log *mlog.Log, l []adns.TLSA) []adns.TLSA {
func filterUsableTLSARecords(log mlog.Log, l []adns.TLSA) []adns.TLSA {
// Gather "usable" records. ../rfc/7672:708
o := 0
for _, r := range l {
@ -368,12 +376,12 @@ func filterUsableTLSARecords(log *mlog.Log, l []adns.TLSA) []adns.TLSA {
}
case adns.TLSAMatchTypeSHA256:
if len(r.CertAssoc) != sha256.Size {
log.Debug("dane tlsa record with wrong data size for sha2-256", mlog.Field("got", len(r.CertAssoc)), mlog.Field("expect", sha256.Size))
log.Debug("dane tlsa record with wrong data size for sha2-256", slog.Int("got", len(r.CertAssoc)), slog.Int("expect", sha256.Size))
continue
}
case adns.TLSAMatchTypeSHA512:
if len(r.CertAssoc) != sha512.Size {
log.Debug("dane tlsa record with wrong data size for sha2-512", mlog.Field("got", len(r.CertAssoc)), mlog.Field("expect", sha512.Size))
log.Debug("dane tlsa record with wrong data size for sha2-512", slog.Int("got", len(r.CertAssoc)), slog.Int("expect", sha512.Size))
continue
}
default:

12
smtpclient/gather_test.go
View file

@ -47,7 +47,7 @@ func ipdomains(s ...string) (l []dns.IPDomain) {
// exist or has temporary error.
func TestGatherDestinations(t *testing.T) {
ctxbg := context.Background()
log := mlog.New("smtpclient")
log := mlog.New("smtpclient", nil)
resolver := dns.MockResolver{
MX: map[string][]*net.MX{
@ -89,7 +89,7 @@ func TestGatherDestinations(t *testing.T) {
test := func(ipd dns.IPDomain, expHosts []dns.IPDomain, expDomain dns.Domain, expPerm, expAuthic, expExpAuthic bool, expErr error) {
t.Helper()
_, authic, authicExp, ed, hosts, perm, err := GatherDestinations(ctxbg, log, resolver, ipd)
_, authic, authicExp, ed, hosts, perm, err := GatherDestinations(ctxbg, log.Logger, resolver, ipd)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
// todo: could also check the individual errors? code currently does not have structured errors.
t.Fatalf("gather hosts: %v, expected %v", err, expErr)
@ -134,7 +134,7 @@ func TestGatherDestinations(t *testing.T) {
func TestGatherIPs(t *testing.T) {
ctxbg := context.Background()
log := mlog.New("smtpclient")
log := mlog.New("smtpclient", nil)
resolver := dns.MockResolver{
A: map[string][]string{
@ -164,7 +164,7 @@ func TestGatherIPs(t *testing.T) {
test := func(host dns.IPDomain, expAuthic, expAuthicExp bool, expHostExp dns.Domain, expIPs []net.IP, expErr any) {
t.Helper()
authic, authicExp, hostExp, ips, _, err := GatherIPs(ctxbg, log, resolver, host, nil)
authic, authicExp, hostExp, ips, _, err := GatherIPs(ctxbg, log.Logger, resolver, host, nil)
if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) {
// todo: could also check the individual errors?
t.Fatalf("gather hosts: %v, expected %v", err, expErr)
@ -207,7 +207,7 @@ func TestGatherIPs(t *testing.T) {
func TestGatherTLSA(t *testing.T) {
ctxbg := context.Background()
log := mlog.New("smtpclient")
log := mlog.New("smtpclient", nil)
record := func(usage, selector, matchType uint8) adns.TLSA {
return adns.TLSA{
@ -253,7 +253,7 @@ func TestGatherTLSA(t *testing.T) {
test := func(host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain, expDANERequired bool, expRecords []adns.TLSA, expBaseDom dns.Domain, expErr any) {
t.Helper()
daneReq, records, baseDom, err := GatherTLSA(ctxbg, log, resolver, host, expandedAuthentic, expandedHost)
daneReq, records, baseDom, err := GatherTLSA(ctxbg, log.Logger, resolver, host, expandedAuthentic, expandedHost)
if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) {
// todo: could also check the individual errors?
t.Fatalf("gather tlsa: %v, expected %v", err, expErr)

9
smtpserver/alignment.go
View file

@ -5,6 +5,7 @@ import (
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/spf"
"github.com/mjl-/mox/store"
@ -12,9 +13,9 @@ import (
// Alignment compares the msgFromDomain with the dkim and spf results, and returns
// a validation, one of: Strict, Relaxed, None.
func alignment(ctx context.Context, msgFromDomain dns.Domain, dkimResults []dkim.Result, spfStatus spf.Status, spfIdentity *dns.Domain) store.Validation {
func alignment(ctx context.Context, log mlog.Log, msgFromDomain dns.Domain, dkimResults []dkim.Result, spfStatus spf.Status, spfIdentity *dns.Domain) store.Validation {
var strict, relaxed bool
msgFromOrgDomain := publicsuffix.Lookup(ctx, msgFromDomain)
msgFromOrgDomain := publicsuffix.Lookup(ctx, log.Logger, msgFromDomain)
// todo: should take temperror and permerror into account.
for _, dr := range dkimResults {
@ -25,12 +26,12 @@ func alignment(ctx context.Context, msgFromDomain dns.Domain, dkimResults []dkim
strict = true
break
} else {
relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, dr.Sig.Domain)
relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, log.Logger, dr.Sig.Domain)
}
}
if !strict && spfStatus == spf.StatusPass {
strict = msgFromDomain == *spfIdentity
relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, *spfIdentity)
relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, log.Logger, *spfIdentity)
}
if strict {
return store.ValidationStrict

46
smtpserver/analyze.go
View file

@ -8,6 +8,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dkim"
@ -89,7 +91,7 @@ func isListDomain(d delivery, ld dns.Domain) bool {
return false
}
func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis {
func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis {
var headers string
mailbox := d.rcptAcc.destination.Mailbox
@ -158,7 +160,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
d.m.MailboxID = mb.ID
d.m.MailboxDestinedID = mb.ID
} else {
log.Debug("mailbox not found in database", mlog.Field("mailbox", mailbox))
log.Debug("mailbox not found in database", slog.String("mailbox", mailbox))
}
return nil
}
@ -206,17 +208,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
if d.dmarcResult.Status != dmarc.StatusPass {
log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")
headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"
} else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
} else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
log.Infox("parsing dmarc aggregate report", err)
headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"
} else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {
log.Infox("parsing domain in dmarc aggregate report", err)
headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"
} else if _, ok := mox.Conf.Domain(d); !ok {
log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d))
log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d))
headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"
} else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {
log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"
} else {
dmarcReport = report
@ -230,7 +232,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
matchesDomain := func(sigDomain dns.Domain) bool {
// RFC seems to require exact DKIM domain match with submitt and message From, we
// also allow msgFrom to be subdomain. ../rfc/8460:322
return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, sigDomain)
return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain)
}
// Valid DKIM signature for domain must be present. We take "valid" to assume
// "passing", not "syntactically valid". We also check for "tlsrpt" as service.
@ -255,13 +257,13 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
if !ok {
log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report")
headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"
} else if report, err := tlsrpt.ParseMessage(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
} else if report, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
log.Infox("parsing tls report", err)
headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"
} else {
var known bool
for _, p := range report.Policies {
log.Info("tlsrpt policy domain", mlog.Field("domain", p.Policy.Domain))
log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain))
if d, err := dns.ParseDomain(p.Policy.Domain); err != nil {
log.Infox("parsing domain in tls report", err)
} else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain {
@ -297,10 +299,10 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
})
})
if err != nil {
log.Infox("determining reputation", err, mlog.Field("message", d.m))
log.Infox("determining reputation", err, slog.Any("message", d.m))
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError)
}
log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method)))
log.Info("reputation analyzed", slog.Bool("conclusive", conclusive), slog.Any("isjunk", isjunk), slog.String("method", string(method)))
if conclusive {
if !*isjunk {
return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers}
@ -340,9 +342,9 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
log.Errorx("get key for verifying subject token", err)
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError)
}
err = subjectpass.Verify(log, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)
err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period)
pass := err == nil
log.Infox("pass by subject token", err, mlog.Field("pass", pass))
log.Infox("pass by subject token", err, slog.Bool("pass", pass))
if pass {
return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers}
}
@ -395,11 +397,11 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
reason = reasonJunkContent
if suspiciousIPrevFail && threshold > 0.25 {
threshold = 0.25
log.Info("setting junk threshold due to iprev fail", mlog.Field("threshold", threshold))
log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold))
reason = reasonJunkContentStrict
} else if !d.tls && threshold > 0.25 {
threshold = 0.25
log.Info("setting junk threshold due to plaintext smtp", mlog.Field("threshold", threshold))
log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold))
reason = reasonJunkContentStrict
} else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) {
// A common theme in junk messages is your recipient address not being in the To/Cc
@ -407,12 +409,12 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
// providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be
// sent with matching Bcc headers. We don't get here for known senders.
threshold = 0.25
log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", mlog.Field("threshold", threshold))
log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold))
reason = reasonJunkContentStrict
}
accept = contentProb <= threshold
junkSubjectpass = contentProb < threshold-0.2
log.Info("content analyzed", mlog.Field("accept", accept), mlog.Field("contentprob", contentProb), mlog.Field("subjectpass", junkSubjectpass))
log.Info("content analyzed", slog.Bool("accept", accept), slog.Float64("contentprob", contentProb), slog.Bool("subjectpass", junkSubjectpass))
} else if err != store.ErrNoJunkFilter {
log.Errorx("open junkfilter", err)
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError)
@ -426,18 +428,18 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
blocked := func(zone dns.Domain) bool {
dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second)
defer dnsblcancel()
if !checkDNSBLHealth(dnsblctx, resolver, zone) {
log.Info("dnsbl not healthy, skipping", mlog.Field("zone", zone))
if !checkDNSBLHealth(dnsblctx, log, resolver, zone) {
log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone))
return false
}
status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(d.m.RemoteIP))
status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP))
dnsblcancel()
if status == dnsbl.StatusFail {
log.Info("rejecting due to listing in dnsbl", mlog.Field("zone", zone), mlog.Field("explanation", expl))
log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl))
return true
} else if err != nil {
log.Infox("dnsbl lookup", err, mlog.Field("zone", zone), mlog.Field("status", status))
log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status))
}
return false
}
@ -459,7 +461,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {
log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation")
pass := subjectpass.Generate(d.msgFrom, []byte(subjectpassKey), time.Now())
pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now())
return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass)
}

5
smtpserver/dnsbl.go
View file

@ -8,6 +8,7 @@ import (
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dnsbl"
"github.com/mjl-/mox/mlog"
)
var dnsblHealth = struct {
@ -23,12 +24,12 @@ type dnsblStatus struct {
}
// checkDNSBLHealth checks healthiness of DNSBL "zone", keeping the result cached for 4 hours.
func checkDNSBLHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rok bool) {
func checkDNSBLHealth(ctx context.Context, log mlog.Log, resolver dns.Resolver, zone dns.Domain) (rok bool) {
dnsblHealth.Lock()
defer dnsblHealth.Unlock()
status, ok := dnsblHealth.zones[zone]
if !ok || time.Since(status.last) > 4*time.Hour {
status.err = dnsbl.CheckHealth(ctx, resolver, zone)
status.err = dnsbl.CheckHealth(ctx, log.Logger, resolver, zone)
status.last = time.Now()
dnsblHealth.zones[zone] = status
}

2
smtpserver/dsn.go
View file

@ -24,7 +24,7 @@ func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, req
}
}
f, err := store.CreateMessageTemp("smtp-dsn")
f, err := store.CreateMessageTemp(c.log, "smtp-dsn")
if err != nil {
return fmt.Errorf("creating temp file: %w", err)
}

6
smtpserver/fuzz_test.go
View file

@ -9,6 +9,7 @@ import (
"time"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/store"
@ -30,17 +31,18 @@ func FuzzServer(f *testing.F) {
f.Add("NOOP")
f.Add("QUIT")
log := mlog.New("smtpserver", nil)
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/smtpserverfuzz/mox.conf")
mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
os.RemoveAll(dataDir)
acc, err := store.OpenAccount("mjl")
acc, err := store.OpenAccount(log, "mjl")
if err != nil {
f.Fatalf("open account: %v", err)
}
defer acc.Close()
err = acc.SetPassword("testtest")
err = acc.SetPassword(log, "testtest")
if err != nil {
f.Fatalf("set password: %v", err)
}

8
smtpserver/rejects.go
View file

@ -8,6 +8,8 @@ import (
"io"
"os"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
@ -17,15 +19,15 @@ import (
)
// rejectPresent returns whether the message is already present in the rejects mailbox.
func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *store.Message, f *os.File) (present bool, msgID string, hash []byte, rerr error) {
if p, err := message.Parse(log, false, store.FileMsgReader(m.MsgPrefix, f)); err != nil {
func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *store.Message, f *os.File) (present bool, msgID string, hash []byte, rerr error) {
if p, err := message.Parse(log.Logger, false, store.FileMsgReader(m.MsgPrefix, f)); err != nil {
log.Infox("parsing reject message for message-id", err)
} else if header, err := p.Header(); err != nil {
log.Infox("parsing reject message header for message-id", err)
} else {
msgID, _, err = message.MessageIDCanonical(header.Get("Message-Id"))
if err != nil {
log.Debugx("parsing message-id for reject", err, mlog.Field("messageid", header.Get("Message-Id")))
log.Debugx("parsing message-id for reject", err, slog.String("messageid", header.Get("Message-Id")))
}
}

6
smtpserver/reputation.go
View file

@ -5,6 +5,8 @@ import (
"fmt"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/mlog"
@ -96,7 +98,7 @@ const (
// ../rfc/6376:1915
// ../rfc/6376:3716
// ../rfc/7208:2167
func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rconclusive bool, rmethod reputationMethod, rerr error) {
func reputation(tx *bstore.Tx, log mlog.Log, m *store.Message) (rjunk *bool, rconclusive bool, rmethod reputationMethod, rerr error) {
boolptr := func(v bool) *bool {
return &v
}
@ -141,7 +143,7 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc
xmessageList := func(q *bstore.Query[store.Message], descr string) []store.Message {
t0 := time.Now()
l, err := q.List()
log.Debugx("querying messages for reputation", err, mlog.Field("msgs", len(l)), mlog.Field("descr", descr), mlog.Field("queryduration", time.Since(t0)))
log.Debugx("querying messages for reputation", err, slog.Int("msgs", len(l)), slog.String("descr", descr), slog.Duration("queryduration", time.Since(t0)))
if err != nil {
panic(queryError(fmt.Sprintf("listing messages: %v", err)))
}

11
smtpserver/reputation_test.go
View file

@ -11,11 +11,14 @@ import (
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/store"
)
var pkglog = mlog.New("smtpserver", nil)
func TestReputation(t *testing.T) {
boolptr := func(v bool) *bool {
return &v
@ -26,6 +29,8 @@ func TestReputation(t *testing.T) {
now := time.Now()
var uidgen store.UID
log := mlog.New("smtpserver", nil)
message := func(junk bool, ageDays int, ehlo, mailfrom, msgfrom, rcptto string, msgfromvalidation store.Validation, dkimDomains []string, mailfromValid, ehloValid bool, ip string) store.Message {
mailFromValidation := store.ValidationNone
@ -77,7 +82,7 @@ func TestReputation(t *testing.T) {
MsgFromLocalpart: msgFrom.Localpart,
MsgFromDomain: msgFrom.Domain.Name(),
MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, msgFrom.Domain).Name(),
MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, log.Logger, msgFrom.Domain).Name(),
MailFromValidated: mailfromValid,
EHLOValidated: ehloValid,
@ -119,7 +124,7 @@ func TestReputation(t *testing.T) {
rcptToDomain, err := dns.ParseDomain(hm.RcptToDomain)
tcheck(t, err, "parse rcptToDomain")
rcptToOrgDomain := publicsuffix.Lookup(ctxbg, rcptToDomain)
rcptToOrgDomain := publicsuffix.Lookup(ctxbg, log.Logger, rcptToDomain)
r := store.Recipient{MessageID: hm.ID, Localpart: hm.RcptToLocalpart, Domain: hm.RcptToDomain, OrgDomain: rcptToOrgDomain.Name(), Sent: hm.Received}
err = tx.Insert(&r)
tcheck(t, err, "insert recipient")
@ -136,7 +141,7 @@ func TestReputation(t *testing.T) {
var method reputationMethod
err = db.Read(ctxbg, func(tx *bstore.Tx) error {
var err error
isjunk, conclusive, method, err = reputation(tx, xlog, &m)
isjunk, conclusive, method, err = reputation(tx, pkglog, &m)
return err
})
tcheck(t, err, "read tx")

179
smtpserver/server.go
View file

@ -27,6 +27,7 @@ import (
"time"
"golang.org/x/exp/maps"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -57,10 +58,6 @@ import (
"github.com/mjl-/mox/tlsrptdb"
)
// Most logging should be done through conn.log* functions.
// Only use log in contexts without connection.
var xlog = mlog.New("smtpserver")
// We use panic and recover for error handling while executing commands.
// These errors signal the connection must be closed.
var errIO = errors.New("io error")
@ -233,14 +230,15 @@ func Listen() {
var servers []func()
func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig *tls.Config, submission, xtls bool, maxMessageSize int64, requireTLSForAuth, requireTLSForDelivery, requireTLS bool, dnsBLs []dns.Domain, firstTimeSenderDelay time.Duration) {
log := mlog.New("smtpserver", nil)
addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
if os.Getuid() == 0 {
xlog.Print("listening for smtp", mlog.Field("listener", name), mlog.Field("address", addr), mlog.Field("protocol", protocol))
log.Print("listening for smtp", slog.String("listener", name), slog.String("address", addr), slog.String("protocol", protocol))
}
network := mox.Network(ip)
ln, err := mox.Listen(network, addr)
if err != nil {
xlog.Fatalx("smtp: listen for smtp", err, mlog.Field("protocol", protocol), mlog.Field("listener", name))
log.Fatalx("smtp: listen for smtp", err, slog.String("protocol", protocol), slog.String("listener", name))
}
if xtls {
ln = tls.NewListener(ln, tlsConfig)
@ -250,10 +248,12 @@ func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig
for {
conn, err := ln.Accept()
if err != nil {
xlog.Infox("smtp: accept", err, mlog.Field("protocol", protocol), mlog.Field("listener", name))
log.Infox("smtp: accept", err, slog.String("protocol", protocol), slog.String("listener", name))
continue
}
resolver := dns.StrictResolver{} // By leaving Pkg empty, it'll be set by each package that uses the resolver, e.g. spf/dkim/dmarc.
// Package is set on the resolver by the dkim/spf/dmarc/etc packages.
resolver := dns.StrictResolver{Log: log.Logger}
go serve(name, mox.Cid(), hostname, tlsConfig, conn, resolver, submission, xtls, maxMessageSize, requireTLSForAuth, requireTLSForDelivery, requireTLS, dnsBLs, firstTimeSenderDelay)
}
}
@ -292,7 +292,7 @@ type conn struct {
localIP net.IP
remoteIP net.IP
hostname dns.Domain
log *mlog.Log
log mlog.Log
maxMessageSize int64
requireTLSForAuth bool
requireTLSForDelivery bool // If set, delivery is only allowed with TLS (STARTTLS), except if delivery is to a TLS reporting address.
@ -378,7 +378,7 @@ func (c *conn) xcheckAuth() {
}
}
func (c *conn) xtrace(level mlog.Level) func() {
func (c *conn) xtrace(level slog.Level) func() {
c.xflush()
c.tr.SetTrace(level)
c.tw.SetTrace(level)
@ -458,7 +458,7 @@ func (c *conn) Read(buf []byte) (int, error) {
var bufpool = moxio.NewBufpool(8, 2*1024)
func (c *conn) readline() string {
line, err := bufpool.Readline(c.r)
line, err := bufpool.Readline(c.log, c.r)
if err != nil && errors.Is(err, moxio.ErrLineTooLong) {
c.writecodeline(smtp.C500BadSyntax, smtp.SeProto5Other0, "line too long, smtp max is 512, we reached 2048", nil)
panic(fmt.Errorf("%s (%w)", err, errIO))
@ -476,7 +476,7 @@ func (c *conn) bwritecodeline(code int, secode string, msg string, err error) {
ecode = fmt.Sprintf("%d.%s", code/100, secode)
}
metricCommands.WithLabelValues(c.kind(), c.cmd, fmt.Sprintf("%d", code), ecode).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second))
c.log.Debugx("smtp command result", err, mlog.Field("kind", c.kind()), mlog.Field("cmd", c.cmd), mlog.Field("code", fmt.Sprintf("%d", code)), mlog.Field("ecode", ecode), mlog.Field("duration", time.Since(c.cmdStart)))
c.log.Debugx("smtp command result", err, slog.String("kind", c.kind()), slog.String("cmd", c.cmd), slog.Int("code", code), slog.String("ecode", ecode), slog.Duration("duration", time.Since(c.cmdStart)))
var sep string
if ecode != "" {
@ -563,15 +563,18 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
dnsBLs: dnsBLs,
firstTimeSenderDelay: firstTimeSenderDelay,
}
c.log = xlog.MoreFields(func() []mlog.Pair {
var logmutex sync.Mutex
c.log = mlog.New("smtpserver", nil).WithFunc(func() []slog.Attr {
logmutex.Lock()
defer logmutex.Unlock()
now := time.Now()
l := []mlog.Pair{
mlog.Field("cid", c.cid),
mlog.Field("delta", now.Sub(c.lastlog)),
l := []slog.Attr{
slog.Int64("cid", c.cid),
slog.Duration("delta", now.Sub(c.lastlog)),
}
c.lastlog = now
if c.username != "" {
l = append(l, mlog.Field("username", c.username))
l = append(l, slog.String("username", c.username))
}
return l
})
@ -581,7 +584,7 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
c.w = bufio.NewWriter(c.tw)
metricConnection.WithLabelValues(c.kind()).Inc()
c.log.Info("new connection", mlog.Field("remote", c.conn.RemoteAddr()), mlog.Field("local", c.conn.LocalAddr()), mlog.Field("submission", submission), mlog.Field("tls", tls), mlog.Field("listener", listenerName))
c.log.Info("new connection", slog.Any("remote", c.conn.RemoteAddr()), slog.Any("local", c.conn.LocalAddr()), slog.Bool("submission", submission), slog.Bool("tls", tls), slog.String("listener", listenerName))
defer func() {
c.origConn.Close() // Close actual TCP socket, regardless of TLS on top.
@ -599,7 +602,7 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
} else if err, ok := x.(error); ok && isClosed(err) {
c.log.Infox("connection closed", err)
} else {
c.log.Error("unhandled panic", mlog.Field("err", x))
c.log.Error("unhandled panic", slog.Any("err", x))
debug.PrintStack()
metrics.PanicInc(metrics.Smtpserver)
}
@ -621,13 +624,13 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
// If remote IP/network resulted in too many authentication failures, refuse to serve.
if submission && !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) {
metrics.AuthenticationRatelimitedInc("submission")
c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP))
c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP))
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many auth failures", nil)
return
}
if !limiterConnections.Add(c.remoteIP, time.Now(), 1) {
c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP))
c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP))
c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many open connections from your ip or network", nil)
return
}
@ -892,7 +895,7 @@ func (c *conn) cmdStarttls(p *parser) {
}
cancel()
tlsversion, ciphersuite := mox.TLSInfo(tlsConn)
c.log.Debug("tls server handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite))
c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite))
c.conn = tlsConn
c.tr = moxio.NewTraceReader(c.log, "RC: ", c)
c.tw = moxio.NewTraceWriter(c.log, "LS: ", c)
@ -1030,11 +1033,11 @@ func (c *conn) cmdAuth(p *parser) {
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "cannot assume other role")
}
acc, err := store.OpenEmailAuth(authc, password)
acc, err := store.OpenEmailAuth(c.log, authc, password)
if err != nil && errors.Is(err, store.ErrUnknownCredentials) {
// ../rfc/4954:274
authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", authc), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", authc), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
}
xcheckf(err, "verifying credentials")
@ -1075,11 +1078,11 @@ func (c *conn) cmdAuth(p *parser) {
password := string(xreadContinuation())
c.xtrace(mlog.LevelTrace) // Restore.
acc, err := store.OpenEmailAuth(username, password)
acc, err := store.OpenEmailAuth(c.log, username, password)
if err != nil && errors.Is(err, store.ErrUnknownCredentials) {
// ../rfc/4954:274
authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", username), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
}
xcheckf(err, "verifying credentials")
@ -1107,11 +1110,11 @@ func (c *conn) cmdAuth(p *parser) {
xsmtpUserErrorf(smtp.C501BadParamSyntax, smtp.SeProto5BadParams4, "malformed cram-md5 response")
}
addr := t[0]
c.log.Debug("cram-md5 auth", mlog.Field("address", addr))
acc, _, err := store.OpenEmail(addr)
c.log.Debug("cram-md5 auth", slog.String("address", addr))
acc, _, err := store.OpenEmail(c.log, addr)
if err != nil {
if errors.Is(err, store.ErrUnknownCredentials) {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
}
}
@ -1127,7 +1130,7 @@ func (c *conn) cmdAuth(p *parser) {
err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
password, err := bstore.QueryTx[store.Password](tx).Get()
if err == bstore.ErrAbsent {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
}
if err != nil {
@ -1141,8 +1144,8 @@ func (c *conn) cmdAuth(p *parser) {
xcheckf(err, "tx read")
})
if ipadhash == nil || opadhash == nil {
c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", mlog.Field("username", addr))
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", addr))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
}
@ -1151,7 +1154,7 @@ func (c *conn) cmdAuth(p *parser) {
opadhash.Write(ipadhash.Sum(nil))
digest := fmt.Sprintf("%x", opadhash.Sum(nil))
if digest != t[1] {
c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
}
@ -1181,13 +1184,13 @@ func (c *conn) cmdAuth(p *parser) {
c0 := xreadInitial()
ss, err := scram.NewServer(h, c0)
xcheckf(err, "starting scram")
c.log.Debug("scram auth", mlog.Field("authentication", ss.Authentication))
acc, _, err := store.OpenEmail(ss.Authentication)
c.log.Debug("scram auth", slog.String("authentication", ss.Authentication))
acc, _, err := store.OpenEmail(c.log, ss.Authentication)
if err != nil {
// todo: we could continue scram with a generated salt, deterministically generated
// from the username. that way we don't have to store anything but attackers cannot
// learn if an account exists. same for absent scram saltedpassword below.
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible")
}
defer func() {
@ -1209,8 +1212,8 @@ func (c *conn) cmdAuth(p *parser) {
xscram = password.SCRAMSHA256
}
if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) {
c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", mlog.Field("address", ss.Authentication))
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP))
c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("address", ss.Authentication))
c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible")
}
xcheckf(err, "fetching credentials")
@ -1230,7 +1233,7 @@ func (c *conn) cmdAuth(p *parser) {
c.readline() // Should be "*" for cancellation.
if errors.Is(err, scram.ErrInvalidProof) {
authResult = "badcreds"
c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP))
c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP))
xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad credentials")
}
xcheckf(err, "server final")
@ -1398,11 +1401,11 @@ func (c *conn) cmdMail(p *parser) {
if c.submission && (len(rpath.IPDomain.IP) > 0 || !rpathAllowed()) {
// ../rfc/6409:522
c.log.Info("submission with unconfigured mailfrom", mlog.Field("user", c.username), mlog.Field("mailfrom", rpath.String()))
c.log.Info("submission with unconfigured mailfrom", slog.String("user", c.username), slog.String("mailfrom", rpath.String()))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user")
} else if !c.submission && len(rpath.IPDomain.IP) > 0 {
// todo future: allow if the IP is the same as this connection is coming from? does later code allow this?
c.log.Info("delivery from address without domain", mlog.Field("mailfrom", rpath.String()))
c.log.Info("delivery from address without domain", slog.String("mailfrom", rpath.String()))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7Other0, "domain name required")
}
@ -1494,7 +1497,7 @@ func (c *conn) cmdRcpt(p *parser) {
cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid)
spfctx, spfcancel := context.WithTimeout(cidctx, time.Minute)
defer spfcancel()
receivedSPF, _, _, _, err := spf.Verify(spfctx, c.resolver, spfArgs)
receivedSPF, _, _, _, err := spf.Verify(spfctx, c.log.Logger, c.resolver, spfArgs)
spfcancel()
if err != nil {
c.log.Errorx("spf verify for multiple recipients", err)
@ -1542,7 +1545,7 @@ func (c *conn) cmdRcpt(p *parser) {
// note: not local for !c.submission is the signal this address is in error.
c.recipients = append(c.recipients, rcptAccount{fpath, false, "", config.Destination{}, ""})
} else {
c.log.Errorx("looking up account for delivery", err, mlog.Field("rcptto", fpath))
c.log.Errorx("looking up account for delivery", err, slog.Any("rcptto", fpath))
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "error processing")
}
c.bwritecodeline(smtp.C250Completed, smtp.SeAddr1Other0, "now on the list", nil)
@ -1583,7 +1586,7 @@ func (c *conn) cmdData(p *parser) {
defer c.xtrace(mlog.LevelTracedata)()
// We read the data into a temporary file. We limit the size and do basic analysis while reading.
dataFile, err := store.CreateMessageTemp("smtp-deliver")
dataFile, err := store.CreateMessageTemp(c.log, "smtp-deliver")
if err != nil {
xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "creating temporary file for message: %s", err)
}
@ -1631,9 +1634,9 @@ func (c *conn) cmdData(p *parser) {
if Localserve && moxvar.Pedantic {
// Require that message can be parsed fully.
p, err := message.Parse(c.log, false, dataFile)
p, err := message.Parse(c.log.Logger, false, dataFile)
if err == nil {
err = p.Walk(c.log, nil)
err = p.Walk(c.log.Logger, nil)
}
if err != nil {
// ../rfc/6409:541
@ -1665,9 +1668,9 @@ func (c *conn) cmdData(p *parser) {
iprevStatus, revName, revNames, iprevAuthentic, err = iprev.Lookup(iprevctx, c.resolver, c.remoteIP)
iprevcancel()
if err != nil {
c.log.Infox("reverse-forward lookup", err, mlog.Field("remoteip", c.remoteIP))
c.log.Infox("reverse-forward lookup", err, slog.Any("remoteip", c.remoteIP))
}
c.log.Debug("dns iprev check", mlog.Field("addr", c.remoteIP), mlog.Field("status", iprevStatus))
c.log.Debug("dns iprev check", slog.Any("addr", c.remoteIP), slog.Any("status", iprevStatus))
var name string
if revName != "" {
name = revName
@ -1721,7 +1724,7 @@ func (c *conn) cmdData(p *parser) {
recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "via", "tcp", "with", with+withComment, "id", mox.ReceivedID(c.cid)) // ../rfc/5321:3158
if c.tls {
tlsConn := c.conn.(*tls.Conn)
tlsComment := message.TLSReceivedComment(c.log, tlsConn.ConnectionState())
tlsComment := mox.TLSReceivedComment(c.log, tlsConn.ConnectionState())
recvHdr.Add(" ", tlsComment...)
}
recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z))
@ -1765,10 +1768,10 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
// for other users.
// We don't check the Sender field, there is no expectation of verification, ../rfc/7489:2948
// and with Resent headers it seems valid to have someone else as Sender. ../rfc/5322:1578
msgFrom, _, header, err := message.From(c.log, true, dataFile)
msgFrom, _, header, err := message.From(c.log.Logger, true, dataFile)
if err != nil {
metricSubmission.WithLabelValues("badmessage").Inc()
c.log.Infox("parsing message From address", err, mlog.Field("user", c.username))
c.log.Infox("parsing message From address", err, slog.String("user", c.username))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SeMsg6Other0, "cannot parse header or From address: %v", err)
}
accName, _, _, err := mox.FindAccount(msgFrom.Localpart, msgFrom.Domain, true)
@ -1778,7 +1781,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
err = mox.ErrAccountNotFound
}
metricSubmission.WithLabelValues("badfrom").Inc()
c.log.Infox("verifying message From address", err, mlog.Field("user", c.username), mlog.Field("msgfrom", msgFrom))
c.log.Infox("verifying message From address", err, slog.String("user", c.username), slog.Any("msgfrom", msgFrom))
xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user")
}
@ -1835,16 +1838,16 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
// Add DKIM signatures.
confDom, ok := mox.Conf.Domain(msgFrom.Domain)
if !ok {
c.log.Error("domain disappeared", mlog.Field("domain", msgFrom.Domain))
c.log.Error("domain disappeared", slog.Any("domain", msgFrom.Domain))
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "internal error")
}
dkimConfig := confDom.DKIM
if len(dkimConfig.Sign) > 0 {
if canonical, err := mox.CanonicalLocalpart(msgFrom.Localpart, confDom); err != nil {
c.log.Errorx("determining canonical localpart for dkim signing", err, mlog.Field("localpart", msgFrom.Localpart))
} else if dkimHeaders, err := dkim.Sign(ctx, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil {
c.log.Errorx("dkim sign for domain", err, mlog.Field("domain", msgFrom.Domain))
c.log.Errorx("determining canonical localpart for dkim signing", err, slog.Any("localpart", msgFrom.Localpart))
} else if dkimHeaders, err := dkim.Sign(ctx, c.log.Logger, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil {
c.log.Errorx("dkim sign for domain", err, slog.Any("domain", msgFrom.Domain))
metricServerErrors.WithLabelValues("dkimsign").Inc()
} else {
msgPrefix = append(msgPrefix, []byte(dkimHeaders)...)
@ -1877,7 +1880,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
mox.Sleep(mox.Context, time.Hour)
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out submission due to special localpart")
} else if code != 0 {
c.log.Info("failure due to special localpart", mlog.Field("code", code))
c.log.Info("failure due to special localpart", slog.Int("code", code))
xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code)
}
}
@ -1894,7 +1897,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err)
}
metricSubmission.WithLabelValues("ok").Inc()
c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize))
c.log.Info("message queued for delivery", slog.Any("mailfrom", *c.mailFrom), slog.Any("rcptto", rcptAcc.rcptTo), slog.Bool("smtputf8", c.smtputf8), slog.Int64("msgsize", msgSize))
err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
xcheckf(err, "adding outgoing message")
@ -1950,7 +1953,7 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) {
mox.Sleep(mox.Context, time.Hour)
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out command due to special localpart")
} else if code != 0 {
c.log.Info("failure due to special localpart", mlog.Field("code", code))
c.log.Info("failure due to special localpart", slog.Int("code", code))
metricDelivery.WithLabelValues("delivererror", "localserve").Inc()
xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code)
}
@ -1961,7 +1964,7 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) {
func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, iprevAuthentic bool, dataFile *os.File) {
// todo: in decision making process, if we run into (some) temporary errors, attempt to continue. if we decide to accept, all good. if we decide to reject, we'll make it a temporary reject.
msgFrom, envelope, headers, err := message.From(c.log, false, dataFile)
msgFrom, envelope, headers, err := message.From(c.log.Logger, false, dataFile)
if err != nil {
c.log.Infox("parsing message for From address", err)
}
@ -2016,7 +2019,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
defer func() {
x := recover() // Should not happen, but don't take program down if it does.
if x != nil {
c.log.Error("dkim verify panic", mlog.Field("err", x))
c.log.Error("dkim verify panic", slog.Any("err", x))
debug.PrintStack()
metrics.PanicInc(metrics.Dkimverify)
}
@ -2029,7 +2032,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
dkimctx, dkimcancel := context.WithTimeout(ctx, time.Minute)
defer dkimcancel()
// todo future: we could let user configure which dkim headers they require
dkimResults, dkimErr = dkim.Verify(dkimctx, c.resolver, c.smtputf8, dkim.DefaultPolicy, dataFile, ignoreTestMode)
dkimResults, dkimErr = dkim.Verify(dkimctx, c.log.Logger, c.resolver, c.smtputf8, dkim.DefaultPolicy, dataFile, ignoreTestMode)
dkimcancel()
}()
@ -2053,7 +2056,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
defer func() {
x := recover() // Should not happen, but don't take program down if it does.
if x != nil {
c.log.Error("spf verify panic", mlog.Field("err", x))
c.log.Error("spf verify panic", slog.Any("err", x))
debug.PrintStack()
metrics.PanicInc(metrics.Spfverify)
}
@ -2061,7 +2064,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
defer wg.Done()
spfctx, spfcancel := context.WithTimeout(ctx, time.Minute)
defer spfcancel()
receivedSPF, spfDomain, spfExpl, spfAuthentic, spfErr = spf.Verify(spfctx, c.resolver, spfArgs)
receivedSPF, spfDomain, spfExpl, spfAuthentic, spfErr = spf.Verify(spfctx, c.log.Logger, c.resolver, spfArgs)
spfcancel()
if spfErr != nil {
c.log.Infox("spf verify", spfErr)
@ -2080,7 +2083,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
if nunknown == len(c.recipients) {
// During RCPT TO we found that the address does not exist.
c.log.Info("deliver attempt to unknown user(s)", mlog.Field("recipients", c.recipients))
c.log.Info("deliver attempt to unknown user(s)", slog.Any("recipients", c.recipients))
// Crude attempt to slow down someone trying to guess names. Would work better
// with connection rate limiter.
@ -2107,7 +2110,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
c.log.Errorx("dkim verify", dkimErr)
authResAddDKIM("none", "", dkimErr.Error(), nil)
} else if len(dkimResults) == 0 {
c.log.Info("no dkim-signature header", mlog.Field("mailfrom", c.mailFrom))
c.log.Info("no dkim-signature header", slog.Any("mailfrom", c.mailFrom))
authResAddDKIM("none", "", "no dkim signatures", nil)
}
for i, r := range dkimResults {
@ -2147,7 +2150,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
errmsg = r.Err.Error()
}
authResAddDKIM(string(r.Status), comment, errmsg, props)
c.log.Debugx("dkim verification result", r.Err, mlog.Field("index", i), mlog.Field("mailfrom", c.mailFrom), mlog.Field("status", r.Status), mlog.Field("domain", domain), mlog.Field("selector", selector), mlog.Field("identity", identity))
c.log.Debugx("dkim verification result", r.Err, slog.Int("index", i), slog.Any("mailfrom", c.mailFrom), slog.Any("status", r.Status), slog.Any("domain", domain), slog.Any("selector", selector), slog.Any("identity", identity))
}
// Add SPF results to Authentication-Results header. ../rfc/7208:2141
@ -2182,7 +2185,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
})
switch receivedSPF.Result {
case spf.StatusPass:
c.log.Debug("spf pass", mlog.Field("ip", spfArgs.RemoteIP), mlog.Field("mailfromdomain", spfArgs.MailFromDomain.ASCII)) // todo: log the domain that was actually verified.
c.log.Debug("spf pass", slog.Any("ip", spfArgs.RemoteIP), slog.String("mailfromdomain", spfArgs.MailFromDomain.ASCII)) // todo: log the domain that was actually verified.
case spf.StatusFail:
if spfExpl != "" {
// Filter out potentially hostile text. ../rfc/7208:2529
@ -2202,14 +2205,14 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if spfExpl == "" {
spfExpl = fmt.Sprintf("your ip %s is not on the SPF allowlist for domain %s", spfArgs.RemoteIP, spfDomain.ASCII)
}
c.log.Info("spf fail", mlog.Field("explanation", spfExpl)) // todo future: get this to the client. how? in smtp session in case of a reject due to dmarc fail?
c.log.Info("spf fail", slog.String("explanation", spfExpl)) // todo future: get this to the client. how? in smtp session in case of a reject due to dmarc fail?
case spf.StatusTemperror:
c.log.Infox("spf temperror", spfErr)
case spf.StatusPermerror:
c.log.Infox("spf permerror", spfErr)
case spf.StatusNone, spf.StatusNeutral, spf.StatusSoftfail:
default:
c.log.Error("unknown spf status, treating as None/Neutral", mlog.Field("status", receivedSPF.Result))
c.log.Error("unknown spf status, treating as None/Neutral", slog.Any("status", receivedSPF.Result))
receivedSPF.Result = spf.StatusNone
}
@ -2228,7 +2231,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
Result: string(dmarcResult.Status),
}
} else {
msgFromValidation = alignment(ctx, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity)
msgFromValidation = alignment(ctx, c.log, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity)
// We are doing the DMARC evaluation now. But we only store it for inclusion in an
// aggregate report when we actually use it. We use an evaluation for each
@ -2241,7 +2244,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute)
defer dmarccancel()
dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage)
dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.log.Logger, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage)
dmarccancel()
var comment string
if dmarcResult.RecordAuthentic {
@ -2265,7 +2268,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507
}
c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain))
c.log.Debug("dmarc verification", slog.Any("result", dmarcResult.Status), slog.Any("domain", msgFrom.Domain))
// Prepare for analyzing content, calculating reputation.
ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP)
@ -2304,13 +2307,13 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
var deliverErrors []deliverError
addError := func(rcptAcc rcptAccount, code int, secode string, userError bool, errmsg string) {
e := deliverError{rcptAcc.rcptTo, code, secode, userError, errmsg}
c.log.Info("deliver error", mlog.Field("rcptto", e.rcptTo), mlog.Field("code", code), mlog.Field("secode", "secode"), mlog.Field("usererror", userError), mlog.Field("errmsg", errmsg))
c.log.Info("deliver error", slog.Any("rcptto", e.rcptTo), slog.Int("code", code), slog.String("secode", "secode"), slog.Bool("usererror", userError), slog.String("errmsg", errmsg))
deliverErrors = append(deliverErrors, e)
}
// For each recipient, do final spam analysis and delivery.
for _, rcptAcc := range c.recipients {
log := c.log.Fields(mlog.Field("mailfrom", c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo))
log := c.log.With(slog.Any("mailfrom", c.mailFrom), slog.Any("rcptto", rcptAcc.rcptTo))
// If this is not a valid local user, we send back a DSN. This can only happen when
// there are also valid recipients, and only when remote is SPF-verified, so the DSN
@ -2326,9 +2329,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
continue
}
acc, err := store.OpenAccount(rcptAcc.accountName)
acc, err := store.OpenAccount(log, rcptAcc.accountName)
if err != nil {
log.Errorx("open account", err, mlog.Field("account", rcptAcc.accountName))
log.Errorx("open account", err, slog.Any("account", rcptAcc.accountName))
metricDelivery.WithLabelValues("accounterror", "").Inc()
addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing")
continue
@ -2347,7 +2350,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
err = acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) {
now := time.Now()
defer func() {
log.Debugx("checking message and size delivery rates", retErr, mlog.Field("duration", time.Since(now)))
log.Debugx("checking message and size delivery rates", retErr, slog.Duration("duration", time.Since(now)))
}()
checkCount := func(msg store.Message, window time.Duration, limit int) {
@ -2440,7 +2443,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
RcptToDomain: rcptAcc.rcptTo.IPDomain.Domain.Name(),
MsgFromLocalpart: msgFrom.Localpart,
MsgFromDomain: msgFrom.Domain.Name(),
MsgFromOrgDomain: publicsuffix.Lookup(ctx, msgFrom.Domain).Name(),
MsgFromOrgDomain: publicsuffix.Lookup(ctx, log.Logger, msgFrom.Domain).Name(),
EHLOValidated: ehloValidation == store.ValidationPass,
MailFromValidated: mailFromValidation == store.ValidationPass,
MsgFromValidated: msgFromValidation == store.ValidationStrict || msgFromValidation == store.ValidationDMARC || msgFromValidation == store.ValidationRelaxed,
@ -2616,7 +2619,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// We'll include all signatures for the organizational domain, even if they weren't
// relevant due to strict alignment requirement.
for _, dkimResult := range dkimResults {
if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, msgFrom.Domain) != publicsuffix.Lookup(ctx, dkimResult.Sig.Domain) {
if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, log.Logger, msgFrom.Domain) != publicsuffix.Lookup(ctx, log.Logger, dkimResult.Sig.Domain) {
continue
}
r := dmarcrpt.DKIMAuthResult{
@ -2684,7 +2687,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
}
log.Info("incoming message rejected", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom))
log.Info("incoming message rejected", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom))
metricDelivery.WithLabelValues("reject", a.reason).Inc()
c.setSlow(true)
addError(rcptAcc, a.code, a.secode, a.userError, a.errmsg)
@ -2704,7 +2707,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
}
if a.tlsReport != nil {
// todo future: add rate limiting to prevent DoS attacks.
if err := tlsrptdb.AddReport(ctx, msgFrom.Domain, c.mailFrom.String(), rcptAcc.destination.HostTLSReports, a.tlsReport); err != nil {
if err := tlsrptdb.AddReport(ctx, c.log, msgFrom.Domain, c.mailFrom.String(), rcptAcc.destination.HostTLSReports, a.tlsReport); err != nil {
log.Errorx("saving TLSRPT report in database", err)
} else {
log.Info("tlsrpt report processed")
@ -2717,13 +2720,13 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
// delivering. If this turns out to be a spammer, we've kept one of their
// connections busy.
if delayFirstTime && !m.IsForward && a.reason == reasonNoBadSignals && c.firstTimeSenderDelay > 0 {
log.Debug("delaying before delivering from sender without reputation", mlog.Field("delay", c.firstTimeSenderDelay))
log.Debug("delaying before delivering from sender without reputation", slog.Duration("delay", c.firstTimeSenderDelay))
mox.Sleep(mox.Context, c.firstTimeSenderDelay)
}
// Gather the message-id before we deliver and the file may be consumed.
if !parsedMessageID {
if p, err := message.Parse(c.log, false, store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil {
if p, err := message.Parse(c.log.Logger, false, store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil {
log.Infox("parsing message for message-id", err)
} else if header, err := p.Header(); err != nil {
log.Infox("parsing message header for message-id", err)
@ -2735,11 +2738,11 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if Localserve {
code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart)
if timeout {
c.log.Info("timing out due to special localpart")
log.Info("timing out due to special localpart")
mox.Sleep(mox.Context, time.Hour)
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeOther00}, "timing out delivery due to special localpart")
} else if code != 0 {
c.log.Info("failure due to special localpart", mlog.Field("code", code))
log.Info("failure due to special localpart", slog.Int("code", code))
metricDelivery.WithLabelValues("delivererror", "localserve").Inc()
addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code))
}
@ -2752,12 +2755,12 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
return
}
metricDelivery.WithLabelValues("delivered", a.reason).Inc()
log.Info("incoming message delivered", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom))
log.Info("incoming message delivered", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom))
conf, _ := acc.Conf()
if conf.RejectsMailbox != "" && m.MessageID != "" {
if err := acc.RejectsRemove(log, conf.RejectsMailbox, m.MessageID); err != nil {
log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID))
log.Errorx("removing message from rejects mailbox", err, slog.String("messageid", messageID))
}
}
})

26
smtpserver/server_test.go
View file

@ -23,6 +23,8 @@ import (
"testing"
"time"
"golang.org/x/exp/slog"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/config"
@ -106,15 +108,16 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test
dmarcdb.EvalDB = nil
}
log := mlog.New("smtpserver", nil)
mox.Context = ctxbg
mox.ConfigStaticPath = configPath
mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
os.RemoveAll(dataDir)
var err error
ts.acc, err = store.OpenAccount("mjl")
ts.acc, err = store.OpenAccount(log, "mjl")
tcheck(t, err, "open account")
err = ts.acc.SetPassword("testtest")
err = ts.acc.SetPassword(log, "testtest")
tcheck(t, err, "set password")
ts.switchStop = store.Switchboard()
err = queue.Init()
@ -169,7 +172,8 @@ func (ts *testserver) run(fn func(helloErr error, client *smtpclient.Client)) {
Auth: auth,
RootCAs: mox.Conf.Static.TLS.CertPool,
}
client, err := smtpclient.New(ctxbg, xlog.WithCid(ts.cid-1), clientConn, ts.tlsmode, ts.tlspkix, ourHostname, remoteHostname, opts)
log := pkglog.WithCid(ts.cid - 1)
client, err := smtpclient.New(ctxbg, log.Logger, clientConn, ts.tlsmode, ts.tlspkix, ourHostname, remoteHostname, opts)
if err != nil {
clientConn.Close()
} else {
@ -355,13 +359,13 @@ func TestDelivery(t *testing.T) {
}
func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) {
mf, err := store.CreateMessageTemp("queue-dsn")
mf, err := store.CreateMessageTemp(pkglog, "queue-dsn")
tcheck(t, err, "temp message")
defer os.Remove(mf.Name())
defer mf.Close()
_, err = mf.Write([]byte(msg))
tcheck(t, err, "write message")
err = acc.DeliverMailbox(xlog, mailbox, m, mf)
err = acc.DeliverMailbox(pkglog, mailbox, m, mf)
tcheck(t, err, "deliver message")
err = mf.Close()
tcheck(t, err, "close message")
@ -376,7 +380,7 @@ func tretrain(t *testing.T, acc *store.Account) {
bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom")
os.Remove(dbPath)
os.Remove(bloomPath)
jf, _, err := acc.OpenJunkFilter(ctxbg, xlog)
jf, _, err := acc.OpenJunkFilter(ctxbg, pkglog)
tcheck(t, err, "open junk filter")
defer jf.Close()
@ -1004,7 +1008,7 @@ func TestTLSReport(t *testing.T) {
tcheck(t, xerr, "write msg")
msg := msgb.String()
headers, xerr := dkim.Sign(ctxbg, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg))
headers, xerr := dkim.Sign(ctxbg, pkglog.Logger, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg))
tcheck(t, xerr, "dkim sign")
msg = headers + msg
@ -1040,7 +1044,7 @@ func TestRatelimitConnectionrate(t *testing.T) {
// We'll be creating 300 connections, no TLS and reduce noise.
ts.tlsmode = smtpclient.TLSSkip
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelInfo})
mlog.SetConfig(map[string]slog.Level{"": mlog.LevelInfo})
// We may be passing a window boundary during this tests. The limit is 300/minute.
// So make twice that many connections and hope the tests don't take too long.
@ -1272,7 +1276,7 @@ func TestCatchall(t *testing.T) {
tcheck(t, err, "checking delivered messages")
tcompare(t, n, 3)
acc, err := store.OpenAccount("catchall")
acc, err := store.OpenAccount(pkglog, "catchall")
tcheck(t, err, "open account")
defer acc.Close()
n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count()
@ -1361,7 +1365,7 @@ test email
f, err := queue.OpenMessage(ctxbg, msgs[0].ID)
tcheck(t, err, "open message in queue")
defer f.Close()
results, err := dkim.Verify(ctxbg, resolver, false, dkim.DefaultPolicy, f, false)
results, err := dkim.Verify(ctxbg, pkglog.Logger, resolver, false, dkim.DefaultPolicy, f, false)
tcheck(t, err, "verifying dkim message")
tcompare(t, len(results), 1)
tcompare(t, results[0].Status, dkim.StatusPass)
@ -1504,7 +1508,7 @@ test email
tcheck(t, err, "listing queue")
tcompare(t, len(msgs), 1)
tcompare(t, msgs[0].RequireTLS, expRequireTLS)
_, err = queue.Drop(ctxbg, msgs[0].ID, "", "")
_, err = queue.Drop(ctxbg, pkglog, msgs[0].ID, "", "")
tcheck(t, err, "deleting message from queue")
})
}

42
spf/spf.go
View file

@ -16,6 +16,8 @@ import (
"strings"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -28,8 +30,6 @@ import (
// sure we make names absolute when looking up. For verifying, we do not want to
// verify names relative to our local search domain.
var xlog = mlog.New("spf")
var (
metricSPFVerify = promauto.NewHistogramVec(
prometheus.HistogramOpts{
@ -129,11 +129,11 @@ var timeNow = time.Now
// Lookup looks up and parses an SPF TXT record for domain.
//
// authentic indicates if the DNS results were DNSSEC-verified.
func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rstatus Status, rtxt string, rrecord *Record, authentic bool, rerr error) {
log := xlog.WithContext(ctx)
func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rstatus Status, rtxt string, rrecord *Record, authentic bool, rerr error) {
log := mlog.New("spf", elog)
start := time.Now()
defer func() {
log.Debugx("spf lookup result", rerr, mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start)))
log.Debugx("spf lookup result", rerr, slog.Any("domain", domain), slog.Any("status", rstatus), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start)))
}()
// ../rfc/7208:586
@ -194,12 +194,12 @@ func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rsta
// of 2 lookups resulting in no records ("void lookups").
//
// authentic indicates if the DNS results were DNSSEC-verified.
func Verify(ctx context.Context, resolver dns.Resolver, args Args) (received Received, domain dns.Domain, explanation string, authentic bool, rerr error) {
log := xlog.WithContext(ctx)
func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, args Args) (received Received, domain dns.Domain, explanation string, authentic bool, rerr error) {
log := mlog.New("spf", elog)
start := time.Now()
defer func() {
metricSPFVerify.WithLabelValues(string(received.Result)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("spf verify result", rerr, mlog.Field("domain", args.domain), mlog.Field("ip", args.RemoteIP), mlog.Field("status", received.Result), mlog.Field("explanation", explanation), mlog.Field("duration", time.Since(start)))
log.Debugx("spf verify result", rerr, slog.Any("domain", args.domain), slog.Any("ip", args.RemoteIP), slog.Any("status", received.Result), slog.String("explanation", explanation), slog.Duration("duration", time.Since(start)))
}()
isHello, ok := prepare(&args)
@ -215,7 +215,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, args Args) (received Rec
return received, dns.Domain{}, "", false, nil
}
status, mechanism, expl, authentic, err := checkHost(ctx, resolver, args)
status, mechanism, expl, authentic, err := checkHost(ctx, log, resolver, args)
comment := fmt.Sprintf("domain %s", args.domain.ASCII)
if isHello {
comment += ", from ehlo because mailfrom is empty"
@ -272,37 +272,35 @@ func prepare(args *Args) (isHello bool, ok bool) {
}
// lookup spf record, then evaluate args against it.
func checkHost(ctx context.Context, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
status, _, record, rauthentic, err := Lookup(ctx, resolver, args.domain)
func checkHost(ctx context.Context, log mlog.Log, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
status, _, record, rauthentic, err := Lookup(ctx, log.Logger, resolver, args.domain)
if err != nil {
return status, "", "", rauthentic, err
}
var evalAuthentic bool
rstatus, mechanism, rexplanation, evalAuthentic, rerr = evaluate(ctx, record, resolver, args)
rstatus, mechanism, rexplanation, evalAuthentic, rerr = evaluate(ctx, log, record, resolver, args)
rauthentic = rauthentic && evalAuthentic
return
}
// Evaluate evaluates the IP and names from args against the SPF DNS record for the domain.
func Evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
func Evaluate(ctx context.Context, elog *slog.Logger, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
log := mlog.New("spf", elog)
_, ok := prepare(&args)
if !ok {
return StatusNone, "default", "", false, fmt.Errorf("no domain name to validate")
}
return evaluate(ctx, record, resolver, args)
return evaluate(ctx, log, record, resolver, args)
}
// evaluate RemoteIP against domain from args, given record.
func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
log := xlog.WithContext(ctx)
func evaluate(ctx context.Context, log mlog.Log, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) {
start := time.Now()
defer func() {
log.Debugx("spf evaluate result", rerr, mlog.Field("dnsrequests", *args.dnsRequests), mlog.Field("voidlookups", *args.voidLookups), mlog.Field("domain", args.domain), mlog.Field("status", rstatus), mlog.Field("mechanism", mechanism), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start)))
log.Debugx("spf evaluate result", rerr, slog.Int("dnsrequests", *args.dnsRequests), slog.Int("voidlookups", *args.voidLookups), slog.Any("domain", args.domain), slog.Any("status", rstatus), slog.String("mechanism", mechanism), slog.String("explanation", rexplanation), slog.Duration("duration", time.Since(start)))
}()
resolver = dns.WithPackage(resolver, "spf")
if args.dnsRequests == nil {
args.dnsRequests = new(int)
args.voidLookups = new(int)
@ -388,7 +386,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A
nargs := args
nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")}
nargs.explanation = &record.Explanation // ../rfc/7208:1548
status, _, _, authentic, err := checkHost(ctx, resolver, nargs)
status, _, _, authentic, err := checkHost(ctx, log, resolver, nargs)
rauthentic = rauthentic && authentic
// ../rfc/7208:1202
switch status {
@ -477,7 +475,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A
for _, rname := range rnames {
rd, err := dns.ParseDomain(strings.TrimSuffix(rname, "."))
if err != nil {
log.Errorx("bad address in ptr record", err, mlog.Field("address", rname))
log.Errorx("bad address in ptr record", err, slog.String("address", rname))
continue
}
// ../rfc/7208-eid4751 ../rfc/7208:1323
@ -565,7 +563,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A
nargs := args
nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")}
nargs.explanation = nil // ../rfc/7208:1548
status, mechanism, expl, authentic, err := checkHost(ctx, resolver, nargs)
status, mechanism, expl, authentic, err := checkHost(ctx, log, resolver, nargs)
rauthentic = rauthentic && authentic
if status == StatusNone {
return StatusPermerror, mechanism, "", rauthentic, err

15
spf/spf_test.go
View file

@ -10,9 +10,12 @@ import (
"time"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp"
)
var pkglog = mlog.New("spf", nil)
func TestLookup(t *testing.T) {
resolver := dns.MockResolver{
TXT: map[string][]string{
@ -31,7 +34,7 @@ func TestLookup(t *testing.T) {
t.Helper()
d := dns.Domain{ASCII: domain}
status, txt, record, _, err := Lookup(context.Background(), resolver, d)
status, txt, record, _, err := Lookup(context.Background(), pkglog.Logger, resolver, d)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %v, expected err %v", err, expErr)
}
@ -259,7 +262,7 @@ func TestVerify(t *testing.T) {
LocalIP: xip("127.0.0.1"),
LocalHostname: dns.Domain{ASCII: "localhost"},
}
received, _, _, _, err := Verify(ctx, r, args)
received, _, _, _, err := Verify(ctx, pkglog.Logger, r, args)
if received.Result != status {
t.Fatalf("got status %q, expected %q, for ip %q (err %v)", received.Result, status, ip, err)
}
@ -345,7 +348,7 @@ func TestVerifyMultipleDomain(t *testing.T) {
LocalIP: net.ParseIP("127.0.0.1"),
LocalHostname: dns.Domain{ASCII: "localhost"},
}
received, _, _, _, err := Verify(context.Background(), resolver, args)
received, _, _, _, err := Verify(context.Background(), pkglog.Logger, resolver, args)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
@ -370,7 +373,7 @@ func TestVerifyScenarios(t *testing.T) {
test := func(resolver dns.Resolver, args Args, expStatus Status, expDomain string, expExpl string, expErr error) {
t.Helper()
recv, d, expl, _, err := Verify(context.Background(), resolver, args)
recv, d, expl, _, err := Verify(context.Background(), pkglog.Logger, resolver, args)
if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
t.Fatalf("got err %v, expected %v", err, expErr)
}
@ -505,7 +508,7 @@ func TestEvaluate(t *testing.T) {
record := &Record{}
resolver := dns.MockResolver{}
args := Args{}
status, _, _, _, _ := Evaluate(context.Background(), record, resolver, args)
status, _, _, _, _ := Evaluate(context.Background(), pkglog.Logger, record, resolver, args)
if status != StatusNone {
t.Fatalf("got status %q, expected none", status)
}
@ -513,7 +516,7 @@ func TestEvaluate(t *testing.T) {
args = Args{
HelloDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "test.example"}},
}
status, mechanism, _, _, err := Evaluate(context.Background(), record, resolver, args)
status, mechanism, _, _, err := Evaluate(context.Background(), pkglog.Logger, record, resolver, args)
if status != StatusNeutral || mechanism != "default" || err != nil {
t.Fatalf("got status %q, mechanism %q, err %v, expected neutral, default, no error", status, mechanism, err)
}

93
store/account.go
View file

@ -44,6 +44,7 @@ import (
"golang.org/x/crypto/bcrypt"
"golang.org/x/exp/slices"
"golang.org/x/exp/slog"
"golang.org/x/text/unicode/norm"
"github.com/mjl-/bstore"
@ -67,8 +68,6 @@ import (
// false again.
var CheckConsistencyOnClose = true
var xlog = mlog.New("store")
var (
ErrUnknownMailbox = errors.New("no such mailbox")
ErrUnknownCredentials = errors.New("credentials not found")
@ -577,15 +576,15 @@ func (m *Message) PrepareExpunge() {
// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the
// envelope in part.
func (m *Message) PrepareThreading(log *mlog.Log, part *message.Part) {
func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {
if part.Envelope == nil {
return
}
messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)
if err != nil {
log.Debugx("parsing message-id, ignoring", err, mlog.Field("messageid", part.Envelope.MessageID))
log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))
} else if raw {
log.Debug("could not parse message-id as address, continuing with raw value", mlog.Field("messageid", part.Envelope.MessageID))
log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))
}
m.MessageID = messageID
m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)
@ -747,7 +746,7 @@ func closeAccount(acc *Account) (rerr error) {
//
// No additional data path prefix or ".db" suffix should be added to the name.
// A single shared account exists per name.
func OpenAccount(name string) (*Account, error) {
func OpenAccount(log mlog.Log, name string) (*Account, error) {
openAccounts.Lock()
defer openAccounts.Unlock()
if acc, ok := openAccounts.names[name]; ok {
@ -759,7 +758,7 @@ func OpenAccount(name string) (*Account, error) {
return nil, ErrAccountUnknown
}
acc, err := openAccount(name)
acc, err := openAccount(log, name)
if err != nil {
return nil, err
}
@ -768,15 +767,15 @@ func OpenAccount(name string) (*Account, error) {
}
// openAccount opens an existing account, or creates it if it is missing.
func openAccount(name string) (a *Account, rerr error) {
func openAccount(log mlog.Log, name string) (a *Account, rerr error) {
dir := filepath.Join(mox.DataDirPath("accounts"), name)
return OpenAccountDB(dir, name)
return OpenAccountDB(log, dir, name)
}
// OpenAccountDB opens an account database file and returns an initialized account
// or error. Only exported for use by subcommands that verify the database file.
// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.
func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {
dbpath := filepath.Join(accountDir, "index.db")
// Create account if it doesn't exist yet.
@ -823,7 +822,7 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {
if !mentioned {
mentioned = true
xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName))
log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))
}
mc, err := mb.CalculateCounts(tx)
if err != nil {
@ -866,17 +865,17 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
// Ensure all messages have a MessageID and SubjectBase, which are needed when
// matching threads.
// Then assign messages to threads, in the same way we do during imports.
xlog.Info("upgrading account for threading, in background", mlog.Field("account", acc.Name))
log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))
go func() {
defer func() {
err := closeAccount(acc)
xlog.Check(err, "closing use of account after upgrading account storage for threads", mlog.Field("account", a.Name))
log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))
}()
defer func() {
x := recover() // Should not happen, but don't take program down if it does.
if x != nil {
xlog.Error("upgradeThreads panic", mlog.Field("err", x))
log.Error("upgradeThreads panic", slog.Any("err", x))
debug.PrintStack()
metrics.PanicInc(metrics.Upgradethreads)
acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)
@ -886,12 +885,12 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
close(acc.threadsCompleted)
}()
err := upgradeThreads(mox.Shutdown, acc, &up)
err := upgradeThreads(mox.Shutdown, log, acc, &up)
if err != nil {
a.threadsErr = err
xlog.Errorx("upgrading account for threading, aborted", err, mlog.Field("account", a.Name))
log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))
} else {
xlog.Info("upgrading account for threading, completed", mlog.Field("account", a.Name))
log.Info("upgrading account for threading, completed", slog.String("account", a.Name))
}
}()
return acc, nil
@ -901,7 +900,7 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
// account has completed, and returns an error if not successful.
//
// To be used before starting an import of messages.
func (a *Account) ThreadingWait(log *mlog.Log) error {
func (a *Account) ThreadingWait(log mlog.Log) error {
select {
case <-a.threadsCompleted:
return a.threadsErr
@ -1224,7 +1223,7 @@ func (a *Account) WithRLock(fn func()) {
// Caller must broadcast new message.
//
// Caller must update mailbox counts.
func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error {
func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error {
if m.Expunged {
return fmt.Errorf("cannot deliver expunged message")
}
@ -1245,9 +1244,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.
var part *message.Part
if m.ParsedBuf == nil {
p, err := message.EnsurePart(log, false, mr, m.Size)
p, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil {
log.Infox("parsing delivered message", err, mlog.Field("parse", ""), mlog.Field("message", m.ID))
log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))
// We continue, p is still valid.
}
part = &p
@ -1259,7 +1258,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
} else {
var p message.Part
if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
log.Errorx("unmarshal parsed message, continuing", err, mlog.Field("parse", ""))
log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))
} else {
part = &p
}
@ -1328,19 +1327,19 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
for _, addr := range addrs {
if addr.User == "" {
// Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.
log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", mlog.Field("address", addr))
log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))
continue
}
d, err := dns.ParseDomain(addr.Host)
if err != nil {
log.Debugx("parsing domain in to/cc/bcc address", err, mlog.Field("address", addr))
log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))
continue
}
mr := Recipient{
MessageID: m.ID,
Localpart: smtp.Localpart(addr.User),
Domain: d.Name(),
OrgDomain: publicsuffix.Lookup(context.TODO(), d).Name(),
OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),
Sent: sent,
}
if err := tx.Insert(&mr); err != nil {
@ -1365,9 +1364,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
}
if sync {
if err := moxio.SyncDir(msgDir); err != nil {
if err := moxio.SyncDir(log, msgDir); err != nil {
xerr := os.Remove(msgPath)
log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath))
log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
return fmt.Errorf("sync directory: %w", err)
}
}
@ -1376,7 +1375,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
l := []Message{*m}
if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {
xerr := os.Remove(msgPath)
log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath))
log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
return fmt.Errorf("training junkfilter: %w", err)
}
*m = l[0]
@ -1387,7 +1386,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
// SetPassword saves a new password for this account. This password is used for
// IMAP, SMTP (submission) sessions and the HTTP account web page.
func (a *Account) SetPassword(password string) error {
func (a *Account) SetPassword(log mlog.Log, password string) error {
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return fmt.Errorf("generating password hash: %w", err)
@ -1439,7 +1438,7 @@ func (a *Account) SetPassword(password string) error {
return nil
})
if err == nil {
xlog.Info("new password set for account", mlog.Field("account", a.Name))
log.Info("new password set for account", slog.String("account", a.Name))
}
return err
}
@ -1590,21 +1589,21 @@ func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, erro
// MessageRuleset returns the first ruleset (if any) that message the message
// represented by msgPrefix and msgFile, with smtp and validation fields from m.
func MessageRuleset(log *mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {
func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {
if len(dest.Rulesets) == 0 {
return nil
}
mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.
p, err := message.Parse(log, false, mr)
p, err := message.Parse(log.Logger, false, mr)
if err != nil {
log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, mlog.Field("parse", ""))
log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))
// note: part is still set.
}
// todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.
header, err := p.Header()
if err != nil {
log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, mlog.Field("parse", ""))
log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))
// todo: reject message?
return nil
}
@ -1678,7 +1677,7 @@ func (a *Account) MessageReader(m Message) *MsgReader {
// Caller must hold account wlock (mailbox may be created).
// Message delivery, possible mailbox creation, and updated mailbox counts are
// broadcasted.
func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {
func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {
var mailbox string
rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)
if rs != nil {
@ -1696,7 +1695,7 @@ func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m *
// Caller must hold account wlock (mailbox may be created).
// Message delivery, possible mailbox creation, and updated mailbox counts are
// broadcasted.
func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgFile *os.File) error {
func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {
var changes []Change
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
@ -1734,7 +1733,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF
//
// Caller most hold account wlock.
// Changes are broadcasted.
func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {
func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {
var changes []Change
var remove []Message
@ -1742,7 +1741,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
for _, m := range remove {
p := a.MessagePath(m.ID)
err := os.Remove(p)
log.Check(err, "removing rejects message file", mlog.Field("path", p))
log.Check(err, "removing rejects message file", slog.String("path", p))
}
}()
@ -1796,7 +1795,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
return hasSpace, nil
}
func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
if len(l) == 0 {
return nil, nil
}
@ -1858,7 +1857,7 @@ func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx *
// RejectsRemove removes a message from the rejects mailbox if present.
// Caller most hold account wlock.
// Changes are broadcasted.
func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string) error {
func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {
var changes []Change
var remove []Message
@ -1866,7 +1865,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string)
for _, m := range remove {
p := a.MessagePath(m.ID)
err := os.Remove(p)
log.Check(err, "removing rejects message file", mlog.Field("path", p))
log.Check(err, "removing rejects message file", slog.String("path", p))
}
}()
@ -1933,8 +1932,8 @@ func manageAuthCache() {
// OpenEmailAuth opens an account given an email address and password.
//
// The email address may contain a catchall separator.
func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
acc, _, rerr = OpenEmail(email)
func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {
acc, _, rerr = OpenEmail(log, email)
if rerr != nil {
return
}
@ -1942,7 +1941,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
defer func() {
if rerr != nil && acc != nil {
err := acc.Close()
xlog.Check(err, "closing account after open auth failure")
log.Check(err, "closing account after open auth failure")
acc = nil
}
}()
@ -1973,7 +1972,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
// OpenEmail opens an account given an email address.
//
// The email address may contain a catchall separator.
func OpenEmail(email string) (*Account, config.Destination, error) {
func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {
addr, err := smtp.ParseAddress(email)
if err != nil {
return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)
@ -1984,7 +1983,7 @@ func OpenEmail(email string) (*Account, config.Destination, error) {
} else if err != nil {
return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)
}
acc, err := OpenAccount(accountName)
acc, err := OpenAccount(log, accountName)
if err != nil {
return nil, config.Destination{}, err
}
@ -2383,7 +2382,7 @@ func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (chang
// indicates that and an error is returned.
//
// Caller should broadcast the changes and remove files for the removed message IDs.
func (a *Account) MailboxDelete(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
// Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about
// NoInferior and NoSelect. We just require only leaf mailboxes are deleted.
qmb := bstore.QueryTx[Mailbox](tx)

34
store/account_test.go
View file

@ -19,6 +19,7 @@ import (
)
var ctxbg = context.Background()
var pkglog = mlog.New("store", nil)
func tcheck(t *testing.T, err error, msg string) {
t.Helper()
@ -28,10 +29,11 @@ func tcheck(t *testing.T, err error, msg string) {
}
func TestMailbox(t *testing.T) {
log := mlog.New("store", nil)
os.RemoveAll("../testdata/store/data")
mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf")
mox.MustLoadConfig(true, false)
acc, err := OpenAccount("mjl")
acc, err := OpenAccount(log, "mjl")
tcheck(t, err, "open account")
defer func() {
err = acc.Close()
@ -39,9 +41,7 @@ func TestMailbox(t *testing.T) {
}()
defer Switchboard()()
log := mlog.New("store")
msgFile, err := CreateMessageTemp("account-test")
msgFile, err := CreateMessageTemp(log, "account-test")
if err != nil {
t.Fatalf("creating temp msg file: %s", err)
}
@ -72,7 +72,7 @@ func TestMailbox(t *testing.T) {
}
acc.WithWLock(func() {
conf, _ := acc.Conf()
err := acc.DeliverDestination(xlog, conf.Destinations["mjl"], &m, msgFile)
err := acc.DeliverDestination(log, conf.Destinations["mjl"], &m, msgFile)
tcheck(t, err, "deliver without consume")
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
@ -81,7 +81,7 @@ func TestMailbox(t *testing.T) {
tcheck(t, err, "sent mailbox")
msent.MailboxID = mbsent.ID
msent.MailboxOrigID = mbsent.ID
err = acc.DeliverMessage(xlog, tx, &msent, msgFile, true, false, false)
err = acc.DeliverMessage(pkglog, tx, &msent, msgFile, true, false, false)
tcheck(t, err, "deliver message")
if !msent.ThreadMuted || !msent.ThreadCollapsed {
t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m")
@ -97,7 +97,7 @@ func TestMailbox(t *testing.T) {
tcheck(t, err, "insert rejects mailbox")
mreject.MailboxID = mbrejects.ID
mreject.MailboxOrigID = mbrejects.ID
err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, true, false, false)
err = acc.DeliverMessage(pkglog, tx, &mreject, msgFile, true, false, false)
tcheck(t, err, "deliver message")
err = tx.Get(&mbrejects)
@ -110,7 +110,7 @@ func TestMailbox(t *testing.T) {
})
tcheck(t, err, "deliver as sent and rejects")
err = acc.DeliverDestination(xlog, conf.Destinations["mjl"], &mconsumed, msgFile)
err = acc.DeliverDestination(pkglog, conf.Destinations["mjl"], &mconsumed, msgFile)
tcheck(t, err, "deliver with consume")
err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
@ -141,7 +141,7 @@ func TestMailbox(t *testing.T) {
})
tcheck(t, err, "untraining non-junk")
err = acc.SetPassword("testtest")
err = acc.SetPassword(log, "testtest")
tcheck(t, err, "set password")
key0, err := acc.Subjectpass("test@localhost")
@ -223,37 +223,37 @@ func TestMailbox(t *testing.T) {
// Run the auth tests twice for possible cache effects.
for i := 0; i < 2; i++ {
_, err := OpenEmailAuth("mjl@mox.example", "bogus")
_, err := OpenEmailAuth(log, "mjl@mox.example", "bogus")
if err != ErrUnknownCredentials {
t.Fatalf("got %v, expected ErrUnknownCredentials", err)
}
}
for i := 0; i < 2; i++ {
acc2, err := OpenEmailAuth("mjl@mox.example", "testtest")
acc2, err := OpenEmailAuth(log, "mjl@mox.example", "testtest")
tcheck(t, err, "open for email with auth")
err = acc2.Close()
tcheck(t, err, "close account")
}
acc2, err := OpenEmailAuth("other@mox.example", "testtest")
acc2, err := OpenEmailAuth(log, "other@mox.example", "testtest")
tcheck(t, err, "open for email with auth")
err = acc2.Close()
tcheck(t, err, "close account")
_, err = OpenEmailAuth("bogus@mox.example", "testtest")
_, err = OpenEmailAuth(log, "bogus@mox.example", "testtest")
if err != ErrUnknownCredentials {
t.Fatalf("got %v, expected ErrUnknownCredentials", err)
}
_, err = OpenEmailAuth("mjl@test.example", "testtest")
_, err = OpenEmailAuth(log, "mjl@test.example", "testtest")
if err != ErrUnknownCredentials {
t.Fatalf("got %v, expected ErrUnknownCredentials", err)
}
}
func TestMessageRuleset(t *testing.T) {
f, err := CreateMessageTemp("msgruleset")
f, err := CreateMessageTemp(pkglog, "msgruleset")
tcheck(t, err, "creating temp msg file")
defer os.Remove(f.Name())
defer f.Close()
@ -284,7 +284,7 @@ Rulesets:
}
dest.Rulesets[0].HeadersRegexpCompiled = hdrs
c := MessageRuleset(xlog, dest, &Message{}, msgBuf, f)
c := MessageRuleset(pkglog, dest, &Message{}, msgBuf, f)
if c == nil {
t.Fatalf("expected ruleset match")
}
@ -293,7 +293,7 @@ Rulesets:
test
`, "\n", "\r\n"))
c = MessageRuleset(xlog, dest, &Message{}, msg2Buf, f)
c = MessageRuleset(pkglog, dest, &Message{}, msg2Buf, f)
if c != nil {
t.Fatalf("expected no ruleset match")
}

8
store/cleanuptemp.go
View file

@ -3,15 +3,17 @@ package store
import (
"os"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/mlog"
)
// CloseRemoveTempFile closes and removes f, a file described by descr. Often
// used in a defer after creating a temporary file.
func CloseRemoveTempFile(log *mlog.Log, f *os.File, descr string) {
func CloseRemoveTempFile(log mlog.Log, f *os.File, descr string) {
name := f.Name()
err := f.Close()
log.Check(err, "closing temporary file", mlog.Field("kind", descr))
log.Check(err, "closing temporary file", slog.String("kind", descr))
err = os.Remove(name)
log.Check(err, "removing temporary file", mlog.Field("kind", descr))
log.Check(err, "removing temporary file", slog.String("kind", descr))
}

Some files were not shown because too many files have changed in this diff Show more