diff --git a/autotls/autotls.go b/autotls/autotls.go index eef25ab..d9f8c2a 100644 --- a/autotls/autotls.go +++ b/autotls/autotls.go @@ -28,9 +28,11 @@ import ( "sync" "time" + "golang.org/x/crypto/acme" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "golang.org/x/crypto/acme" "github.com/mjl-/autocert" @@ -39,8 +41,6 @@ import ( "github.com/mjl-/mox/moxvar" ) -var xlog = mlog.New("autotls") - var ( metricCertput = promauto.NewCounter( prometheus.CounterOpts{ @@ -148,7 +148,7 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h } loggingGetCertificate := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - log := xlog.WithContext(hello.Context()) + log := mlog.New("autotls", nil).WithContext(hello.Context()) // Handle missing SNI to prevent logging an error below. // At startup, during config initialization, we already adjust the tls config to @@ -156,16 +156,16 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h // common for SMTP STARTTLS connections, which often do not care about the // verification of the certificate. if hello.ServerName == "" { - log.Debug("tls request without sni servername, rejecting", mlog.Field("localaddr", hello.Conn.LocalAddr()), mlog.Field("supportedprotos", hello.SupportedProtos)) + log.Debug("tls request without sni servername, rejecting", slog.Any("localaddr", hello.Conn.LocalAddr()), slog.Any("supportedprotos", hello.SupportedProtos)) return nil, fmt.Errorf("sni server name required") } cert, err := m.GetCertificate(hello) if err != nil { if errors.Is(err, errHostNotAllowed) { - log.Debugx("requesting certificate", err, mlog.Field("host", hello.ServerName)) + log.Debugx("requesting certificate", err, slog.String("host", hello.ServerName)) } else { - log.Errorx("requesting certificate", err, mlog.Field("host", hello.ServerName)) + log.Errorx("requesting certificate", err, slog.String("host", hello.ServerName)) } } return cert, err @@ -194,7 +194,7 @@ func Load(name, acmeDir, contactEmail, directoryURL string, getPrivateKey func(h // are fully served by publicIPs (only if non-empty and there is no unspecified // address in the list). If no, log an error with a warning that ACME validation // may fail. -func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.Domain]struct{}, publicIPs []string, checkHosts bool) { +func (m *Manager) SetAllowedHostnames(log mlog.Log, resolver dns.Resolver, hostnames map[dns.Domain]struct{}, publicIPs []string, checkHosts bool) { m.Lock() defer m.Unlock() @@ -207,7 +207,7 @@ func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.D return l[i].Name() < l[j].Name() }) - xlog.Debug("autotls setting allowed hostnames", mlog.Field("hostnames", l), mlog.Field("publicips", publicIPs)) + log.Debug("autotls setting allowed hostnames", slog.Any("hostnames", l), slog.Any("publicips", publicIPs)) var added []dns.Domain for h := range hostnames { if _, ok := m.hosts[h]; !ok { @@ -231,16 +231,16 @@ func (m *Manager) SetAllowedHostnames(resolver dns.Resolver, hostnames map[dns.D publicIPstrs[ip] = struct{}{} } - xlog.Debug("checking ips of hosts configured for acme tls cert validation") + log.Debug("checking ips of hosts configured for acme tls cert validation") for _, h := range added { ips, _, err := resolver.LookupIP(ctx, "ip", h.ASCII+".") if err != nil { - xlog.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, mlog.Field("host", h)) + log.Errorx("warning: acme tls cert validation for host may fail due to dns lookup error", err, slog.Any("host", h)) continue } for _, ip := range ips { if _, ok := publicIPstrs[ip.String()]; !ok { - xlog.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on", mlog.Field("hostname", h), mlog.Field("listenedips", publicIPs), mlog.Field("hostips", ips), mlog.Field("missingip", ip)) + log.Error("warning: acme tls cert validation for host is likely to fail because not all its ips are being listened on", slog.Any("hostname", h), slog.Any("listenedips", publicIPs), slog.Any("hostips", ips), slog.Any("missingip", ip)) } } } @@ -266,9 +266,9 @@ var errHostNotAllowed = errors.New("autotls: host not in allowlist") // present. Only hosts added with SetAllowedHostnames are allowed. During shutdown, // no new connections are allowed. func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) { - log := xlog.WithContext(ctx) + log := mlog.New("autotls", nil).WithContext(ctx) defer func() { - log.WithContext(ctx).Debugx("autotls hostpolicy result", rerr, mlog.Field("host", host)) + log.Debugx("autotls hostpolicy result", rerr, slog.String("host", host)) }() // Don't request new TLS certs when we are shutting down. @@ -300,46 +300,46 @@ func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) { type dirCache autocert.DirCache func (d dirCache) Delete(ctx context.Context, name string) (rerr error) { - log := xlog.WithContext(ctx) + log := mlog.New("autotls", nil).WithContext(ctx) defer func() { - log.Debugx("dircache delete result", rerr, mlog.Field("name", name)) + log.Debugx("dircache delete result", rerr, slog.String("name", name)) }() err := autocert.DirCache(d).Delete(ctx, name) if err != nil { - log.Errorx("deleting cert from dir cache", err, mlog.Field("name", name)) + log.Errorx("deleting cert from dir cache", err, slog.String("name", name)) } else if !strings.HasSuffix(name, "+token") { - log.Info("autotls cert delete", mlog.Field("name", name)) + log.Info("autotls cert delete", slog.String("name", name)) } return err } func (d dirCache) Get(ctx context.Context, name string) (rbuf []byte, rerr error) { - log := xlog.WithContext(ctx) + log := mlog.New("autotls", nil).WithContext(ctx) defer func() { - log.Debugx("dircache get result", rerr, mlog.Field("name", name)) + log.Debugx("dircache get result", rerr, slog.String("name", name)) }() buf, err := autocert.DirCache(d).Get(ctx, name) if err != nil && errors.Is(err, autocert.ErrCacheMiss) { - log.Infox("getting cert from dir cache", err, mlog.Field("name", name)) + log.Infox("getting cert from dir cache", err, slog.String("name", name)) } else if err != nil { - log.Errorx("getting cert from dir cache", err, mlog.Field("name", name)) + log.Errorx("getting cert from dir cache", err, slog.String("name", name)) } else if !strings.HasSuffix(name, "+token") { - log.Debug("autotls cert get", mlog.Field("name", name)) + log.Debug("autotls cert get", slog.String("name", name)) } return buf, err } func (d dirCache) Put(ctx context.Context, name string, data []byte) (rerr error) { - log := xlog.WithContext(ctx) + log := mlog.New("autotls", nil).WithContext(ctx) defer func() { - log.Debugx("dircache put result", rerr, mlog.Field("name", name)) + log.Debugx("dircache put result", rerr, slog.String("name", name)) }() metricCertput.Inc() err := autocert.DirCache(d).Put(ctx, name, data) if err != nil { - log.Errorx("storing cert in dir cache", err, mlog.Field("name", name)) + log.Errorx("storing cert in dir cache", err, slog.String("name", name)) } else if !strings.HasSuffix(name, "+token") { - log.Info("autotls cert store", mlog.Field("name", name)) + log.Info("autotls cert store", slog.String("name", name)) } return err } diff --git a/autotls/autotls_test.go b/autotls/autotls_test.go index 6cf16d1..8b36159 100644 --- a/autotls/autotls_test.go +++ b/autotls/autotls_test.go @@ -12,9 +12,11 @@ import ( "github.com/mjl-/autocert" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" ) func TestAutotls(t *testing.T) { + log := mlog.New("autotls", nil) os.RemoveAll("../testdata/autotls") os.MkdirAll("../testdata/autotls", 0770) @@ -34,7 +36,7 @@ func TestAutotls(t *testing.T) { if err := m.HostPolicy(context.Background(), "mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) { t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err) } - m.SetAllowedHostnames(dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false) + m.SetAllowedHostnames(log, dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false) l = m.Hostnames() if !reflect.DeepEqual(l, []dns.Domain{{ASCII: "mox.example"}}) { t.Fatalf("hostnames, got %v, expected single mox.example", l) @@ -88,7 +90,7 @@ func TestAutotls(t *testing.T) { t.Fatalf("private key changed after reload") } m.shutdown = make(chan struct{}) - m.SetAllowedHostnames(dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false) + m.SetAllowedHostnames(log, dns.StrictResolver{}, map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}}, nil, false) if err := m.HostPolicy(context.Background(), "mox.example"); err != nil { t.Fatalf("hostpolicy, got err %v, expected no error", err) } diff --git a/backup.go b/backup.go index 6a56cb6..462ff6b 100644 --- a/backup.go +++ b/backup.go @@ -12,10 +12,11 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/dmarcdb" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/mtastsdb" @@ -48,51 +49,51 @@ func backupctl(ctx context.Context, ctl *ctl) { writer := ctl.writer() // Format easily readable output for the user. - formatLog := func(prefix, text string, err error, fields ...mlog.Pair) []byte { + formatLog := func(prefix, text string, err error, attrs ...slog.Attr) []byte { var b bytes.Buffer fmt.Fprint(&b, prefix) fmt.Fprint(&b, text) if err != nil { fmt.Fprint(&b, ": "+err.Error()) } - for _, f := range fields { - fmt.Fprintf(&b, "; %s=%v", f.Key, f.Value) + for _, a := range attrs { + fmt.Fprintf(&b, "; %s=%v", a.Key, a.Value) } fmt.Fprint(&b, "\n") return b.Bytes() } // Log an error to both the mox service as the user running "mox backup". - xlogx := func(prefix, text string, err error, fields ...mlog.Pair) { - ctl.log.Errorx(text, err, fields...) + pkglogx := func(prefix, text string, err error, attrs ...slog.Attr) { + ctl.log.Errorx(text, err, attrs...) - _, werr := writer.Write(formatLog(prefix, text, err, fields...)) + _, werr := writer.Write(formatLog(prefix, text, err, attrs...)) ctl.xcheck(werr, "write to ctl") } // Log an error but don't mark backup as failed. - xwarnx := func(text string, err error, fields ...mlog.Pair) { - xlogx("warning: ", text, err, fields...) + xwarnx := func(text string, err error, attrs ...slog.Attr) { + pkglogx("warning: ", text, err, attrs...) } // Log an error that causes the backup to be marked as failed. We typically // continue processing though. - xerrx := func(text string, err error, fields ...mlog.Pair) { + xerrx := func(text string, err error, attrs ...slog.Attr) { incomplete = true - xlogx("error: ", text, err, fields...) + pkglogx("error: ", text, err, attrs...) } // If verbose is enabled, log to the cli command. Always log as info level. - xvlog := func(text string, fields ...mlog.Pair) { - ctl.log.Info(text, fields...) + xvlog := func(text string, attrs ...slog.Attr) { + ctl.log.Info(text, attrs...) if verbose { - _, werr := writer.Write(formatLog("", text, nil, fields...)) + _, werr := writer.Write(formatLog("", text, nil, attrs...)) ctl.xcheck(werr, "write to ctl") } } if _, err := os.Stat(dstDataDir); err == nil { - xwarnx("destination data directory already exists", nil, mlog.Field("dir", dstDataDir)) + xwarnx("destination data directory already exists", nil, slog.String("dir", dstDataDir)) } srcDataDir := filepath.Clean(mox.DataDirPath(".")) @@ -119,7 +120,7 @@ func backupctl(ctx context.Context, ctl *ctl) { sf, err := os.Open(srcpath) if err != nil { - xerrx("open source file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xerrx("open source file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) return } defer sf.Close() @@ -127,7 +128,7 @@ func backupctl(ctx context.Context, ctl *ctl) { ensureDestDir(dstpath) df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) if err != nil { - xerrx("creating destination file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xerrx("creating destination file (not backed up)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) return } defer func() { @@ -136,16 +137,16 @@ func backupctl(ctx context.Context, ctl *ctl) { } }() if _, err := io.Copy(df, sf); err != nil { - xerrx("copying file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xerrx("copying file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) return } err = df.Close() df = nil if err != nil { - xerrx("closing destination file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xerrx("closing destination file (not backed up properly)", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) return } - xvlog("backed up file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmFile))) + xvlog("backed up file", slog.String("path", path), slog.Duration("duration", time.Since(tmFile))) } // Back up the files in a directory (by copying). @@ -155,7 +156,7 @@ func backupctl(ctx context.Context, ctl *ctl) { dstdir := filepath.Join(dstDataDir, dir) err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error { if err != nil { - xerrx("walking file (not backed up)", err, mlog.Field("srcpath", srcpath)) + xerrx("walking file (not backed up)", err, slog.String("srcpath", srcpath)) return nil } if d.IsDir() { @@ -165,10 +166,10 @@ func backupctl(ctx context.Context, ctl *ctl) { return nil }) if err != nil { - xerrx("copying directory (not backed up properly)", err, mlog.Field("srcdir", srcdir), mlog.Field("dstdir", dstdir), mlog.Field("duration", time.Since(tmDir))) + xerrx("copying directory (not backed up properly)", err, slog.String("srcdir", srcdir), slog.String("dstdir", dstdir), slog.Duration("duration", time.Since(tmDir))) return } - xvlog("backed up directory", mlog.Field("dir", dir), mlog.Field("duration", time.Since(tmDir))) + xvlog("backed up directory", slog.String("dir", dir), slog.Duration("duration", time.Since(tmDir))) } // Backup a database by copying it in a readonly transaction. @@ -177,7 +178,7 @@ func backupctl(ctx context.Context, ctl *ctl) { backupDB := func(db *bstore.DB, path string) (rerr error) { defer func() { if rerr != nil { - xerrx("backing up database", rerr, mlog.Field("path", path)) + xerrx("backing up database", rerr, slog.String("path", path)) } }() @@ -216,7 +217,7 @@ func backupctl(ctx context.Context, ctl *ctl) { if err != nil { return fmt.Errorf("closing destination database after copy: %v", err) } - xvlog("backed up database file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmDB))) + xvlog("backed up database file", slog.String("path", path), slog.Duration("duration", time.Since(tmDB))) return nil } @@ -231,7 +232,7 @@ func backupctl(ctx context.Context, ctl *ctl) { // No point in trying with regular copy, we would warn twice. return false, err } else if !warnedHardlink { - xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xwarnx("creating hardlink to message failed, will be doing regular file copies and not warn again", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) warnedHardlink = true } @@ -269,7 +270,7 @@ func backupctl(ctx context.Context, ctl *ctl) { // Start making the backup. tmStart := time.Now() - ctl.log.Print("making backup", mlog.Field("destdir", dstDataDir)) + ctl.log.Print("making backup", slog.String("destdir", dstDataDir)) err := os.MkdirAll(dstDataDir, 0770) if err != nil { @@ -299,14 +300,14 @@ func backupctl(ctx context.Context, ctl *ctl) { tmQueue := time.Now() if err := backupDB(queue.DB, path); err != nil { - xerrx("queue not backed up", err, mlog.Field("path", path), mlog.Field("duration", time.Since(tmQueue))) + xerrx("queue not backed up", err, slog.String("path", path), slog.Duration("duration", time.Since(tmQueue))) return } dstdbpath := filepath.Join(dstDataDir, path) db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...) if err != nil { - xerrx("open copied queue database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmQueue))) + xerrx("open copied queue database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmQueue))) return } @@ -329,7 +330,7 @@ func backupctl(ctx context.Context, ctl *ctl) { srcpath := filepath.Join(srcDataDir, "queue", mp) dstpath := filepath.Join(dstDataDir, "queue", mp) if linked, err := linkOrCopy(srcpath, dstpath); err != nil { - xerrx("linking/copying queue message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xerrx("linking/copying queue message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) } else if linked { nlinked++ } else { @@ -338,9 +339,9 @@ func backupctl(ctx context.Context, ctl *ctl) { return nil }) if err != nil { - xerrx("processing queue messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs))) + xerrx("processing queue messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs))) } else { - xvlog("queue message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs))) + xvlog("queue message files linked/copied", slog.Int("linked", nlinked), slog.Int("copied", ncopied), slog.Duration("duration", time.Since(tmMsgs))) } // Read through all files in queue directory and warn about anything we haven't handled yet. @@ -348,7 +349,7 @@ func backupctl(ctx context.Context, ctl *ctl) { srcqdir := filepath.Join(srcDataDir, "queue") err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error { if err != nil { - xerrx("walking files in queue", err, mlog.Field("srcpath", srcqpath)) + xerrx("walking files in queue", err, slog.String("srcpath", srcqpath)) return nil } if d.IsDir() { @@ -362,17 +363,17 @@ func backupctl(ctx context.Context, ctl *ctl) { return nil } qp := filepath.Join("queue", p) - xwarnx("backing up unrecognized file in queue directory", nil, mlog.Field("path", qp)) + xwarnx("backing up unrecognized file in queue directory", nil, slog.String("path", qp)) backupFile(qp) return nil }) if err != nil { - xerrx("walking queue directory (not backed up properly)", err, mlog.Field("dir", "queue"), mlog.Field("duration", time.Since(tmWalk))) + xerrx("walking queue directory (not backed up properly)", err, slog.String("dir", "queue"), slog.Duration("duration", time.Since(tmWalk))) } else { - xvlog("walked queue directory", mlog.Field("duration", time.Since(tmWalk))) + xvlog("walked queue directory", slog.Duration("duration", time.Since(tmWalk))) } - xvlog("queue backed finished", mlog.Field("duration", time.Since(tmQueue))) + xvlog("queue backed finished", slog.Duration("duration", time.Since(tmQueue))) } backupQueue(filepath.FromSlash("queue/index.db")) @@ -385,7 +386,7 @@ func backupctl(ctx context.Context, ctl *ctl) { dbpath := filepath.Join("accounts", acc.Name, "index.db") err := backupDB(acc.DB, dbpath) if err != nil { - xerrx("copying account database", err, mlog.Field("path", dbpath), mlog.Field("duration", time.Since(tmAccount))) + xerrx("copying account database", err, slog.String("path", dbpath), slog.Duration("duration", time.Since(tmAccount))) } // todo: should document/check not taking a rlock on account. @@ -409,7 +410,7 @@ func backupctl(ctx context.Context, ctl *ctl) { dstdbpath := filepath.Join(dstDataDir, dbpath) db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...) if err != nil { - xerrx("open copied account database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmAccount))) + xerrx("open copied account database", err, slog.String("dstpath", dstdbpath), slog.Duration("duration", time.Since(tmAccount))) return } @@ -433,7 +434,7 @@ func backupctl(ctx context.Context, ctl *ctl) { srcpath := filepath.Join(srcDataDir, amp) dstpath := filepath.Join(dstDataDir, amp) if linked, err := linkOrCopy(srcpath, dstpath); err != nil { - xerrx("linking/copying account message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + xerrx("linking/copying account message", err, slog.String("srcpath", srcpath), slog.String("dstpath", dstpath)) } else if linked { nlinked++ } else { @@ -442,9 +443,9 @@ func backupctl(ctx context.Context, ctl *ctl) { return nil }) if err != nil { - xerrx("processing account messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs))) + xerrx("processing account messages (not backed up properly)", err, slog.Duration("duration", time.Since(tmMsgs))) } else { - xvlog("account message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs))) + xvlog("account message files linked/copied", slog.Int("linked", nlinked), slog.Int("copied", ncopied), slog.Duration("duration", time.Since(tmMsgs))) } // Read through all files in account directory and warn about anything we haven't handled yet. @@ -452,7 +453,7 @@ func backupctl(ctx context.Context, ctl *ctl) { srcadir := filepath.Join(srcDataDir, "accounts", acc.Name) err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error { if err != nil { - xerrx("walking files in account", err, mlog.Field("srcpath", srcapath)) + xerrx("walking files in account", err, slog.String("srcpath", srcapath)) return nil } if d.IsDir() { @@ -472,20 +473,20 @@ func backupctl(ctx context.Context, ctl *ctl) { } ap := filepath.Join("accounts", acc.Name, p) if strings.HasPrefix(p, "msg"+string(filepath.Separator)) { - xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, mlog.Field("path", ap)) + xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, slog.String("path", ap)) } else { - xwarnx("backing up unrecognized file in account directory", nil, mlog.Field("path", ap)) + xwarnx("backing up unrecognized file in account directory", nil, slog.String("path", ap)) } backupFile(ap) return nil }) if err != nil { - xerrx("walking account directory (not backed up properly)", err, mlog.Field("srcdir", srcadir), mlog.Field("duration", time.Since(tmWalk))) + xerrx("walking account directory (not backed up properly)", err, slog.String("srcdir", srcadir), slog.Duration("duration", time.Since(tmWalk))) } else { - xvlog("walked account directory", mlog.Field("duration", time.Since(tmWalk))) + xvlog("walked account directory", slog.Duration("duration", time.Since(tmWalk))) } - xvlog("account backup finished", mlog.Field("dir", filepath.Join("accounts", acc.Name)), mlog.Field("duration", time.Since(tmAccount))) + xvlog("account backup finished", slog.String("dir", filepath.Join("accounts", acc.Name)), slog.Duration("duration", time.Since(tmAccount))) } // For each configured account, open it, make a copy of the database and @@ -493,9 +494,9 @@ func backupctl(ctx context.Context, ctl *ctl) { // account directories when handling "all other files" below. accounts := map[string]struct{}{} for _, accName := range mox.Conf.Accounts() { - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(ctl.log, accName) if err != nil { - xerrx("opening account for copying (will try to copy as regular files later)", err, mlog.Field("account", accName)) + xerrx("opening account for copying (will try to copy as regular files later)", err, slog.String("account", accName)) continue } accounts[accName] = struct{}{} @@ -506,7 +507,7 @@ func backupctl(ctx context.Context, ctl *ctl) { tmWalk := time.Now() err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error { if err != nil { - xerrx("walking path", err, mlog.Field("path", srcpath)) + xerrx("walking path", err, slog.String("path", srcpath)) return nil } @@ -536,18 +537,18 @@ func backupctl(ctx context.Context, ctl *ctl) { return nil case "lastknownversion": // Optional file, not yet handled. default: - xwarnx("backing up unrecognized file", nil, mlog.Field("path", p)) + xwarnx("backing up unrecognized file", nil, slog.String("path", p)) } backupFile(p) return nil }) if err != nil { - xerrx("walking other files (not backed up properly)", err, mlog.Field("duration", time.Since(tmWalk))) + xerrx("walking other files (not backed up properly)", err, slog.Duration("duration", time.Since(tmWalk))) } else { - xvlog("walking other files finished", mlog.Field("duration", time.Since(tmWalk))) + xvlog("walking other files finished", slog.Duration("duration", time.Since(tmWalk))) } - xvlog("backup finished", mlog.Field("duration", time.Since(tmStart))) + xvlog("backup finished", slog.Duration("duration", time.Since(tmStart))) writer.xclose() diff --git a/ctl.go b/ctl.go index e62b299..bb4c068 100644 --- a/ctl.go +++ b/ctl.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/dns" @@ -35,7 +37,7 @@ type ctl struct { conn net.Conn r *bufio.Reader // Set for first reader. x any // If set, errors are handled by calling panic(x) instead of log.Fatal. - log *mlog.Log // If set, along with x, logging is done here. + log mlog.Log // If set, along with x, logging is done here. } // xctl opens a ctl connection. @@ -59,7 +61,7 @@ func (c *ctl) xerror(msg string) { if c.x == nil { log.Fatalln(msg) } - c.log.Debugx("ctl error", fmt.Errorf("%s", msg), mlog.Field("cmd", c.cmd)) + c.log.Debugx("ctl error", fmt.Errorf("%s", msg), slog.String("cmd", c.cmd)) c.xwrite(msg) panic(c.x) } @@ -74,7 +76,7 @@ func (c *ctl) xcheck(err error, msg string) { if c.x == nil { log.Fatalf("%s: %s", msg, err) } - c.log.Debugx(msg, err, mlog.Field("cmd", c.cmd)) + c.log.Debugx(msg, err, slog.String("cmd", c.cmd)) fmt.Fprintf(c.conn, "%s: %s\n", msg, err) panic(c.x) } @@ -160,7 +162,7 @@ type ctlwriter struct { conn net.Conn // Ctl socket from which messages are read. buf []byte // Scratch buffer, for reading response. x any // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal. - log *mlog.Log + log mlog.Log } func (s *ctlwriter) Write(buf []byte) (int, error) { @@ -184,7 +186,7 @@ func (s *ctlwriter) xerror(msg string) { if s.x == nil { log.Fatalln(msg) } else { - s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd)) + s.log.Debugx("error", fmt.Errorf("%s", msg), slog.String("cmd", s.cmd)) panic(s.x) } } @@ -196,7 +198,7 @@ func (s *ctlwriter) xcheck(err error, msg string) { if s.x == nil { log.Fatalf("%s: %s", msg, err) } else { - s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd)) + s.log.Debugx(msg, err, slog.String("cmd", s.cmd)) panic(s.x) } } @@ -213,7 +215,7 @@ type ctlreader struct { err error // If set, returned for each read. can also be io.EOF. npending int // Number of bytes that can still be read until a new count line must be read. x any // If set, errors are handled with panic(x) instead of log.Fatal. - log *mlog.Log // If x is set, logging goes to log. + log mlog.Log // If x is set, logging goes to log. } func (s *ctlreader) Read(buf []byte) (N int, Err error) { @@ -252,7 +254,7 @@ func (s *ctlreader) xerror(msg string) { if s.x == nil { log.Fatalln(msg) } else { - s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd)) + s.log.Debugx("error", fmt.Errorf("%s", msg), slog.String("cmd", s.cmd)) panic(s.x) } } @@ -264,13 +266,13 @@ func (s *ctlreader) xcheck(err error, msg string) { if s.x == nil { log.Fatalf("%s: %s", msg, err) } else { - s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd)) + s.log.Debugx(msg, err, slog.String("cmd", s.cmd)) panic(s.x) } } // servectl handles requests on the unix domain socket "ctl", e.g. for graceful shutdown, local mail delivery. -func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) { +func servectl(ctx context.Context, log mlog.Log, conn net.Conn, shutdown func()) { log.Debug("ctl connection") var stop = struct{}{} // Sentinel value for panic and recover. @@ -280,7 +282,7 @@ func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func() if x == nil || x == stop { return } - log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", ctl.cmd)) + log.Error("servectl panic", slog.Any("err", x), slog.String("cmd", ctl.cmd)) debug.PrintStack() metrics.PanicInc(metrics.Ctl) }() @@ -297,7 +299,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { log := ctl.log cmd := ctl.xread() ctl.cmd = cmd - log.Info("ctl command", mlog.Field("cmd", cmd)) + log.Info("ctl command", slog.String("cmd", cmd)) switch cmd { case "stop": shutdown() @@ -314,10 +316,10 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { */ to := ctl.xread() - a, addr, err := store.OpenEmail(to) + a, addr, err := store.OpenEmail(ctl.log, to) ctl.xcheck(err, "lookup destination address") - msgFile, err := store.CreateMessageTemp("ctl-deliver") + msgFile, err := store.CreateMessageTemp(ctl.log, "ctl-deliver") ctl.xcheck(err, "creating temporary message file") defer store.CloseRemoveTempFile(log, msgFile, "deliver message") mw := message.NewWriter(msgFile) @@ -335,7 +337,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { a.WithWLock(func() { err := a.DeliverDestination(log, addr, m, msgFile) ctl.xcheck(err, "delivering message") - log.Info("message delivered through ctl", mlog.Field("to", to)) + log.Info("message delivered through ctl", slog.Any("to", to)) }) err = a.Close() @@ -353,7 +355,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { account := ctl.xread() pw := ctl.xread() - acc, err := store.OpenAccount(account) + acc, err := store.OpenAccount(ctl.log, account) ctl.xcheck(err, "open account") defer func() { if acc != nil { @@ -362,7 +364,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { } }() - err = acc.SetPassword(pw) + err = acc.SetPassword(ctl.log, pw) ctl.xcheck(err, "setting password") err = acc.Close() ctl.xcheck(err, "closing account") @@ -442,7 +444,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { ctl.xcheck(err, "parsing id") } - count, err := queue.Drop(ctx, id, todomain, recipient) + count, err := queue.Drop(ctx, ctl.log, id, todomain, recipient) ctl.xcheck(err, "dropping messages from queue") ctl.xwrite(fmt.Sprintf("%d", count)) ctl.xwriteok() @@ -586,13 +588,13 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { pkg := ctl.xread() levelstr := ctl.xread() if levelstr == "" { - mox.Conf.LogLevelRemove(pkg) + mox.Conf.LogLevelRemove(ctl.log, pkg) } else { level, ok := mlog.Levels[levelstr] if !ok { ctl.xerror("bad level") } - mox.Conf.LogLevelSet(pkg, level) + mox.Conf.LogLevelSet(ctl.log, pkg, level) } ctl.xwriteok() @@ -603,7 +605,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { < "ok" or error */ account := ctl.xread() - acc, err := store.OpenAccount(account) + acc, err := store.OpenAccount(ctl.log, account) ctl.xcheck(err, "open account") defer func() { if acc != nil { @@ -623,9 +625,9 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { dbPath := filepath.Join(basePath, acc.Name, "junkfilter.db") bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom") err := os.Remove(dbPath) - log.Check(err, "removing old junkfilter database file", mlog.Field("path", dbPath)) + log.Check(err, "removing old junkfilter database file", slog.String("path", dbPath)) err = os.Remove(bloomPath) - log.Check(err, "removing old junkfilter bloom filter file", mlog.Field("path", bloomPath)) + log.Check(err, "removing old junkfilter bloom filter file", slog.String("path", bloomPath)) // Open junk filter, this creates new files. jf, _, err := acc.OpenJunkFilter(ctx, ctl.log) @@ -651,7 +653,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { return err }) ctl.xcheck(err, "training messages") - ctl.log.Info("retrained messages", mlog.Field("total", total), mlog.Field("trained", trained)) + ctl.log.Info("retrained messages", slog.Int("total", total), slog.Int("trained", trained)) // Close junk filter, marking success. err = jf.Close() @@ -668,7 +670,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { < stream */ account := ctl.xread() - acc, err := store.OpenAccount(account) + acc, err := store.OpenAccount(ctl.log, account) ctl.xcheck(err, "open account") defer func() { if acc != nil { @@ -724,7 +726,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { const batchSize = 10000 xfixmsgsize := func(accName string) { - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(ctl.log, accName) ctl.xcheck(err, "open account") defer func() { err := acc.Close() @@ -791,7 +793,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { m.Size = correctSize mr := acc.MessageReader(m) - part, err := message.EnsurePart(log, false, mr, m.Size) + part, err := message.EnsurePart(log.Logger, false, mr, m.Size) if err != nil { _, werr := fmt.Fprintf(w, "parsing message %d again: %v (continuing)\n", m.ID, err) ctl.xcheck(werr, "write") @@ -859,7 +861,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { const batchSize = 100 xreparseAccount := func(accName string) { - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(ctl.log, accName) ctl.xcheck(err, "open account") defer func() { err := acc.Close() @@ -880,7 +882,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { return q.ForEach(func(m store.Message) error { lastID = m.ID mr := acc.MessageReader(m) - p, err := message.EnsurePart(log, false, mr, m.Size) + p, err := message.EnsurePart(log.Logger, false, mr, m.Size) if err != nil { _, err := fmt.Fprintf(w, "parsing message %d: %v (continuing)\n", m.ID, err) ctl.xcheck(err, "write") @@ -935,7 +937,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { w := ctl.writer() xreassignThreads := func(accName string) { - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(ctl.log, accName) ctl.xcheck(err, "open account") defer func() { err := acc.Close() @@ -982,7 +984,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { backupctl(ctx, ctl) default: - log.Info("unrecognized command", mlog.Field("cmd", cmd)) + log.Info("unrecognized command", slog.String("cmd", cmd)) ctl.xwrite("unrecognized command") return } diff --git a/ctl_test.go b/ctl_test.go index 3fd492f..d7001db 100644 --- a/ctl_test.go +++ b/ctl_test.go @@ -21,6 +21,7 @@ import ( ) var ctxbg = context.Background() +var pkglog = mlog.New("ctl", nil) func tcheck(t *testing.T, err error, errmsg string) { if err != nil { @@ -36,19 +37,17 @@ func TestCtl(t *testing.T) { os.RemoveAll("testdata/ctl/data") mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf") mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf") - if errs := mox.LoadConfig(ctxbg, true, false); len(errs) > 0 { + if errs := mox.LoadConfig(ctxbg, pkglog, true, false); len(errs) > 0 { t.Fatalf("loading mox config: %v", errs) } defer store.Switchboard()() - xlog := mlog.New("ctl") - testctl := func(fn func(clientctl *ctl)) { t.Helper() cconn, sconn := net.Pipe() - clientctl := ctl{conn: cconn, log: xlog} - serverctl := ctl{conn: sconn, log: xlog} + clientctl := ctl{conn: cconn, log: pkglog} + serverctl := ctl{conn: sconn, log: pkglog} go servectlcmd(ctxbg, &serverctl, func() {}) fn(&clientctl) cconn.Close() @@ -148,8 +147,8 @@ func TestCtl(t *testing.T) { }) // Export data, import it again - xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, nil) - xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, nil) + xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog}) + xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog}) testctl(func(ctl *ctl) { ctlcmdImport(ctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox")) }) @@ -167,7 +166,7 @@ func TestCtl(t *testing.T) { ctlcmdFixmsgsize(ctl, "mjl") }) testctl(func(ctl *ctl) { - acc, err := store.OpenAccount("mjl") + acc, err := store.OpenAccount(ctl.log, "mjl") tcheck(t, err, "open account") defer acc.Close() @@ -176,13 +175,13 @@ func TestCtl(t *testing.T) { deliver := func(m *store.Message) { t.Helper() m.Size = int64(len(content)) - msgf, err := store.CreateMessageTemp("ctltest") + msgf, err := store.CreateMessageTemp(ctl.log, "ctltest") tcheck(t, err, "create temp file") defer os.Remove(msgf.Name()) defer msgf.Close() _, err = msgf.Write(content) tcheck(t, err, "write message file") - err = acc.DeliverMailbox(xlog, "Inbox", m, msgf) + err = acc.DeliverMailbox(ctl.log, "Inbox", m, msgf) tcheck(t, err, "deliver message") } diff --git a/dane/dane.go b/dane/dane.go index 7e140ca..202ac6c 100644 --- a/dane/dane.go +++ b/dane/dane.go @@ -59,6 +59,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -132,8 +134,8 @@ func (e VerifyError) Unwrap() error { // indicate DNSSEC errors. // - ErrInsecure // - VerifyError, potentially wrapping errors from crypto/x509. -func Dial(ctx context.Context, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) { - log := mlog.New("dane").WithContext(ctx) +func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) { + log := mlog.New("dane", elog) // Split host and port. host, portstr, err := net.SplitHostPort(address) @@ -272,7 +274,7 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a } var verifiedRecord adns.TLSA - config := TLSClientConfig(log, records, baseDom, moreAllowedHosts, &verifiedRecord) + config := TLSClientConfig(log.Logger, records, baseDom, moreAllowedHosts, &verifiedRecord) tlsConn := tls.Client(conn, &config) if err := tlsConn.HandshakeContext(ctx); err != nil { conn.Close() @@ -295,13 +297,14 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a // // If verifiedRecord is not nil, it is set to the record that was successfully // verified, if any. -func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config { +func TLSClientConfig(elog *slog.Logger, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config { + log := mlog.New("dane", elog) return tls.Config{ ServerName: allowedHost.ASCII, // For SNI. InsecureSkipVerify: true, VerifyConnection: func(cs tls.ConnectionState) error { - verified, record, err := Verify(log, records, cs, allowedHost, moreAllowedHosts) - log.Debugx("dane verification", err, mlog.Field("verified", verified), mlog.Field("record", record)) + verified, record, err := Verify(log.Logger, records, cs, allowedHost, moreAllowedHosts) + log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record)) if verified { if verifiedRecord != nil { *verifiedRecord = record @@ -332,7 +335,8 @@ func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain, // If an error is encountered while verifying a record, e.g. for x509 // trusted-anchor verification, an error may be returned, typically one or more // (wrapped) errors of type VerifyError. -func Verify(log *mlog.Log, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) { +func Verify(elog *slog.Logger, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) { + log := mlog.New("dane", elog) metricVerify.Inc() if len(records) == 0 { metricVerifyErrors.Inc() @@ -360,7 +364,7 @@ func Verify(log *mlog.Log, records []adns.TLSA, cs tls.ConnectionState, allowedH // errors while verifying certificates against a trust-anchor, an error can be // returned with one or more underlying x509 verification errors. A nil-nil error // is only returned when verified is false. -func verifySingle(log *mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) { +func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) { if len(cs.PeerCertificates) == 0 { return false, fmt.Errorf("no server certificate") } @@ -513,7 +517,7 @@ func verifySingle(log *mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowed default: // Unknown, perhaps defined in the future. Not an error. - log.Debug("unrecognized tlsa usage, skipping", mlog.Field("tlsausage", tlsa.Usage)) + log.Debug("unrecognized tlsa usage, skipping", slog.Any("tlsausage", tlsa.Usage)) return false, nil } } diff --git a/dane/dane_test.go b/dane/dane_test.go index 753e685..67f1558 100644 --- a/dane/dane_test.go +++ b/dane/dane_test.go @@ -17,9 +17,10 @@ import ( "reflect" "strconv" "sync/atomic" + "testing" "time" - "testing" + "golang.org/x/exp/slog" "github.com/mjl-/adns" @@ -37,7 +38,8 @@ func tcheckf(t *testing.T, err error, format string, args ...any) { // Test dialing and DANE TLS verification. func TestDial(t *testing.T) { - mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) + mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug}) + log := mlog.New("dane", nil) // Create fake CA/trusted-anchor certificate. taTempl := x509.Certificate{ @@ -139,7 +141,7 @@ func TestDial(t *testing.T) { test := func(resolver dns.Resolver, expRecord adns.TLSA, expErr any) { t.Helper() - conn, record, err := Dial(context.Background(), resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages) + conn, record, err := Dial(context.Background(), log.Logger, resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages) if err == nil { conn.Close() } diff --git a/develop.txt b/develop.txt index 5e76101..6d96dbc 100644 --- a/develop.txt +++ b/develop.txt @@ -10,6 +10,14 @@ This file has notes useful for mox developers. name/path manipulation. Do not remove/rename files that are still open. - Not all code uses adns, the DNSSEC-aware resolver. Such as code that makes http requests, like mtasts and autotls/autocert. +- We don't have an internal/ directory, really just to prevent long paths in + the repo, and to keep all Go code matching *.go */*.go (without matching + vendor/). Part of the packages are reusable by other software. Those reusable + packages must not cause mox implementation details (such as bstore) to get out, + which would cause unexpected dependencies. Those packages also only expose the + standard slog package for logging, not our mlog package. Packages not intended + for reuse do use mlog as it is more convenient. Internally, we always use + mlog.Log to do the logging, wrapping an slog.Logger. # TLS certificates diff --git a/dkim/dkim.go b/dkim/dkim.go index 83a34e5..353a9ea 100644 --- a/dkim/dkim.go +++ b/dkim/dkim.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -35,8 +37,6 @@ import ( "github.com/mjl-/mox/smtp" ) -var xlog = mlog.New("dkim") - var ( metricSign = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -123,11 +123,11 @@ type Result struct { // todo: use some io.Writer to hash the body and the header. // Sign returns line(s) with DKIM-Signature headers, generated according to the configuration. -func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) { - log := xlog.WithContext(ctx) +func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) { + log := mlog.New("dkim", elog) start := timeNow() defer func() { - log.Debugx("dkim sign result", rerr, mlog.Field("localpart", localpart), mlog.Field("domain", domain), mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start))) + log.Debugx("dkim sign result", rerr, slog.Any("localpart", localpart), slog.Any("domain", domain), slog.Bool("smtputf8", smtputf8), slog.Duration("duration", time.Since(start))) }() hdrs, bodyOffset, err := parseHeaders(bufio.NewReader(&moxio.AtReader{R: msg})) @@ -270,11 +270,11 @@ func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c co // record should be present. // // authentic indicates if DNS results were DNSSEC-verified. -func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, authentic bool, rerr error) { - log := xlog.WithContext(ctx) +func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, authentic bool, rerr error) { + log := mlog.New("dkim", elog) start := timeNow() defer func() { - log.Debugx("dkim lookup result", rerr, mlog.Field("selector", selector), mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) + log.Debugx("dkim lookup result", rerr, slog.Any("selector", selector), slog.Any("domain", domain), slog.Any("status", rstatus), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start))) }() name := selector.ASCII + "._domainkey." + domain.ASCII + "." @@ -338,8 +338,8 @@ func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Dom // verification failure is treated as actual failure. With ignoreTestMode // false, such verification failures are treated as if there is no signature by // returning StatusNone. -func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) { - log := xlog.WithContext(ctx) +func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) { + log := mlog.New("dkim", elog) start := timeNow() defer func() { duration := float64(time.Since(start)) / float64(time.Second) @@ -353,10 +353,10 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu } if len(results) == 0 { - log.Debugx("dkim verify result", rerr, mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start))) + log.Debugx("dkim verify result", rerr, slog.Bool("smtputf8", smtputf8), slog.Duration("duration", time.Since(start))) } for _, result := range results { - log.Debugx("dkim verify result", result.Err, mlog.Field("smtputf8", smtputf8), mlog.Field("status", result.Status), mlog.Field("sig", result.Sig), mlog.Field("record", result.Record), mlog.Field("duration", time.Since(start))) + log.Debugx("dkim verify result", result.Err, slog.Bool("smtputf8", smtputf8), slog.Any("status", result.Status), slog.Any("sig", result.Sig), slog.Any("record", result.Record), slog.Duration("duration", time.Since(start))) } }() @@ -380,7 +380,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu continue } - h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig) + h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, log, sig) if err != nil { results = append(results, Result{StatusPermerror, sig, nil, false, err}) continue @@ -394,7 +394,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu } br := bufio.NewReader(&moxio.AtReader{R: r, Offset: int64(bodyOffset)}) - status, txt, authentic, err := verifySignature(ctx, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode) + status, txt, authentic, err := verifySignature(ctx, log.Logger, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode) results = append(results, Result{status, sig, txt, authentic, err}) } return results, nil @@ -402,7 +402,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu // check if signature is acceptable. // Only looks at the signature parameters, not at the DNS record. -func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) { +func checkSignatureParams(ctx context.Context, log mlog.Log, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) { // "From" header is required, ../rfc/6376:2122 ../rfc/6376:2546 var from bool for _, h := range sig.SignedHeaders { @@ -431,7 +431,7 @@ func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, cano if subdom.Unicode != "" { subdom.Unicode = "x." + subdom.Unicode } - if orgDom := publicsuffix.Lookup(ctx, subdom); subdom.ASCII == orgDom.ASCII { + if orgDom := publicsuffix.Lookup(ctx, log.Logger, subdom); subdom.ASCII == orgDom.ASCII { return 0, false, false, fmt.Errorf("%w: %s", ErrTLD, sig.Domain) } @@ -480,9 +480,9 @@ func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, cano } // lookup the public key in the DNS and verify the signature. -func verifySignature(ctx context.Context, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, bool, error) { +func verifySignature(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, bool, error) { // ../rfc/6376:2604 - status, record, _, authentic, err := Lookup(ctx, resolver, sig.Selector, sig.Domain) + status, record, _, authentic, err := Lookup(ctx, elog, resolver, sig.Selector, sig.Domain) if err != nil { // todo: for temporary errors, we could pass on information so caller returns a 4.7.5 ecode, ../rfc/6376:2777 return status, nil, authentic, err diff --git a/dkim/dkim_test.go b/dkim/dkim_test.go index 6331f63..34dcd80 100644 --- a/dkim/dkim_test.go +++ b/dkim/dkim_test.go @@ -17,8 +17,11 @@ import ( "github.com/mjl-/mox/config" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" ) +var pkglog = mlog.New("dkim", nil) + func policyOK(sig *Sig) error { return nil } @@ -143,7 +146,7 @@ test }, } - results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false) + results, err := Verify(context.Background(), pkglog.Logger, resolver, false, policyOK, strings.NewReader(message), false) if err != nil { t.Fatalf("dkim verify: %v", err) } @@ -190,7 +193,7 @@ Joe. }, } - results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false) + results, err := Verify(context.Background(), pkglog.Logger, resolver, false, policyOK, strings.NewReader(message), false) if err != nil { t.Fatalf("dkim verify: %v", err) } @@ -262,7 +265,7 @@ test } ctx := context.Background() - headers, err := Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message)) + headers, err := Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message)) if err != nil { t.Fatalf("sign: %v", err) } @@ -293,7 +296,7 @@ test nmsg := headers + message - results, err := Verify(ctx, resolver, false, policyOK, strings.NewReader(nmsg), false) + results, err := Verify(ctx, pkglog.Logger, resolver, false, policyOK, strings.NewReader(nmsg), false) if err != nil { t.Fatalf("verify: %s", err) } @@ -304,31 +307,31 @@ test //log.Infof("nmsg\n%s", nmsg) // Multiple From headers. - _, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: \r\nFrom: \r\n\r\ntest")) + _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: \r\nFrom: \r\n\r\ntest")) if !errors.Is(err, ErrFrom) { t.Fatalf("sign, got err %v, expected ErrFrom", err) } // No From header. - _, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: \r\n\r\ntest")) + _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: \r\n\r\ntest")) if !errors.Is(err, ErrFrom) { t.Fatalf("sign, got err %v, expected ErrFrom", err) } // Malformed headers. - _, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest")) + _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest")) if !errors.Is(err, ErrHeaderMalformed) { t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) } - _, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:\r\n\r\ntest")) + _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:\r\n\r\ntest")) if !errors.Is(err, ErrHeaderMalformed) { t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) } - _, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:\r\n\r\ntest")) + _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:\r\n\r\ntest")) if !errors.Is(err, ErrHeaderMalformed) { t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) } - _, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:")) + _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:")) if !errors.Is(err, ErrHeaderMalformed) { t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) } @@ -408,7 +411,7 @@ test msg = strings.ReplaceAll(msg, "\n", "\r\n") - headers, err := Sign(context.Background(), "mjl", signDomain, dkimConf, false, strings.NewReader(msg)) + headers, err := Sign(context.Background(), pkglog.Logger, "mjl", signDomain, dkimConf, false, strings.NewReader(msg)) if err != nil { t.Fatalf("sign: %v", err) } @@ -425,7 +428,7 @@ test sign() } - results, err := Verify(context.Background(), resolver, true, policy, strings.NewReader(msg), false) + results, err := Verify(context.Background(), pkglog.Logger, resolver, true, policy, strings.NewReader(msg), false) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("got verify error %v, expected %v", err, expErr) } diff --git a/dmarc/dmarc.go b/dmarc/dmarc.go index 50d5dbb..dda9448 100644 --- a/dmarc/dmarc.go +++ b/dmarc/dmarc.go @@ -25,9 +25,9 @@ import ( "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/spf" -) -var xlog = mlog.New("dmarc") + "golang.org/x/exp/slog" +) var ( metricDMARCVerify = promauto.NewHistogramVec( @@ -99,11 +99,11 @@ type Result struct { // domain is the domain with the DMARC record. // // rauthentic indicates if the DNS results were DNSSEC-verified. -func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rauthentic bool, rerr error) { - log := xlog.WithContext(ctx) +func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rauthentic bool, rerr error) { + log := mlog.New("dmarc", elog) start := time.Now() defer func() { - log.Debugx("dmarc lookup result", rerr, mlog.Field("fromdomain", from), mlog.Field("status", status), mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("duration", time.Since(start))) + log.Debugx("dmarc lookup result", rerr, slog.Any("fromdomain", from), slog.Any("status", status), slog.Any("domain", domain), slog.Any("record", record), slog.Duration("duration", time.Since(start))) }() // ../rfc/7489:859 ../rfc/7489:1370 @@ -114,7 +114,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status } if record == nil { // ../rfc/7489:761 ../rfc/7489:1377 - domain = publicsuffix.Lookup(ctx, from) + domain = publicsuffix.Lookup(ctx, log.Logger, from) if domain == from { return StatusNone, domain, nil, txt, authentic, err } @@ -202,11 +202,11 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain // example in RFC 7489. // // authentic indicates if the DNS results were DNSSEC-verified. -func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) { - log := xlog.WithContext(ctx) +func LookupExternalReportsAccepted(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) { + log := mlog.New("dmarc", elog) start := time.Now() defer func() { - log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start))) + log.Debugx("dmarc externalreports result", rerr, slog.Bool("accepts", accepts), slog.Any("dmarcdomain", dmarcDomain), slog.Any("extdestdomain", extDestDomain), slog.Any("records", records), slog.Duration("duration", time.Since(start))) }() status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain) @@ -226,8 +226,8 @@ func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, d // against the message (for inclusion in Authentication-Result headers). // // useResult indicates if the result should be applied in a policy decision. -func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) { - log := xlog.WithContext(ctx) +func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) { + log := mlog.New("dmarc", elog) start := time.Now() defer func() { use := "no" @@ -239,10 +239,10 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes reject = "yes" } metricDMARCVerify.WithLabelValues(string(result.Status), reject, use).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("dmarc verify result", result.Err, mlog.Field("fromdomain", from), mlog.Field("dkimresults", dkimResults), mlog.Field("spfresult", spfResult), mlog.Field("status", result.Status), mlog.Field("reject", result.Reject), mlog.Field("use", useResult), mlog.Field("duration", time.Since(start))) + log.Debugx("dmarc verify result", result.Err, slog.Any("fromdomain", from), slog.Any("dkimresults", dkimResults), slog.Any("spfresult", spfResult), slog.Any("status", result.Status), slog.Bool("reject", result.Reject), slog.Bool("use", useResult), slog.Duration("duration", time.Since(start))) }() - status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from) + status, recordDomain, record, _, authentic, err := Lookup(ctx, log.Logger, resolver, from) if record == nil { return false, Result{false, status, false, false, recordDomain, record, authentic, err} } @@ -277,7 +277,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes if r, ok := pubsuffixes[name]; ok { return r } - r := publicsuffix.Lookup(ctx, name) + r := publicsuffix.Lookup(ctx, log.Logger, name) pubsuffixes[name] = r return r } diff --git a/dmarc/dmarc_test.go b/dmarc/dmarc_test.go index 4e8c617..c95cd3f 100644 --- a/dmarc/dmarc_test.go +++ b/dmarc/dmarc_test.go @@ -8,9 +8,12 @@ import ( "github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/spf" ) +var pkglog = mlog.New("dmarc", nil) + func TestLookup(t *testing.T) { resolver := dns.MockResolver{ TXT: map[string][]string{ @@ -29,7 +32,7 @@ func TestLookup(t *testing.T) { test := func(d string, expStatus Status, expDomain string, expRecord *Record, expErr error) { t.Helper() - status, dom, record, _, _, err := Lookup(context.Background(), resolver, dns.Domain{ASCII: d}) + status, dom, record, _, _, err := Lookup(context.Background(), pkglog.Logger, resolver, dns.Domain{ASCII: d}) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("got err %#v, expected %#v", err, expErr) } @@ -68,7 +71,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) { test := func(dom, extdom string, expStatus Status, expAccepts bool, expErr error) { t.Helper() - accepts, status, _, _, _, err := LookupExternalReportsAccepted(context.Background(), resolver, dns.Domain{ASCII: dom}, dns.Domain{ASCII: extdom}) + accepts, status, _, _, _, err := LookupExternalReportsAccepted(context.Background(), pkglog.Logger, resolver, dns.Domain{ASCII: dom}, dns.Domain{ASCII: extdom}) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("got err %#v, expected %#v", err, expErr) } @@ -124,7 +127,7 @@ func TestVerify(t *testing.T) { if err != nil { t.Fatalf("parsing domain: %v", err) } - useResult, result := Verify(context.Background(), resolver, from, dkimResults, spfResult, spfIdentity, true) + useResult, result := Verify(context.Background(), pkglog.Logger, resolver, from, dkimResults, spfResult, spfIdentity, true) if useResult != expUseResult || !equalResult(result, expResult) { t.Fatalf("verify: got useResult %v, result %#v, expected %v %#v", useResult, result, expUseResult, expResult) } diff --git a/dmarcdb/eval.go b/dmarcdb/eval.go index a11dd6d..1c12dde 100644 --- a/dmarcdb/eval.go +++ b/dmarcdb/eval.go @@ -23,6 +23,7 @@ import ( "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -300,13 +301,13 @@ var jitteredTimeUntil = func(t time.Time) time.Duration { // sends DMARC reports to domains that requested them. func Start(resolver dns.Resolver) { go func() { - log := mlog.New("dmarcdb") + log := mlog.New("dmarcdb", nil) defer func() { // In case of panic don't take the whole program down. x := recover() if x != nil { - log.Error("recover from panic", mlog.Field("panic", x)) + log.Error("recover from panic", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Dmarcdb) } @@ -358,7 +359,7 @@ func Start(resolver dns.Resolver) { log.Check(err, "removing stale dmarc evaluations from database") clog := log.WithCid(mox.Cid()) - clog.Info("sending dmarc aggregate reports", mlog.Field("end", nextEnd.UTC()), mlog.Field("intervals", intervals)) + clog.Info("sending dmarc aggregate reports", slog.Time("end", nextEnd.UTC()), slog.Any("intervals", intervals)) if err := sendReports(ctx, clog, resolver, db, nextEnd, intervals); err != nil { clog.Errorx("sending dmarc aggregate reports", err) metricReportError.Inc() @@ -393,7 +394,7 @@ var sleepBetween = func(ctx context.Context, between time.Duration) (ok bool) { // sendReports gathers all policy domains that have evaluations that should // receive a DMARC report and sends a report to each. -func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error { +func sendReports(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error { ivals := make([]any, len(intervals)) for i, v := range intervals { ivals[i] = v @@ -452,14 +453,14 @@ func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db * // In case of panic don't take the whole program down. x := recover() if x != nil { - log.Error("unhandled panic in dmarcdb sendReports", mlog.Field("panic", x)) + log.Error("unhandled panic in dmarcdb sendReports", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Dmarcdb) } }() defer wg.Done() - rlog := log.WithCid(mox.Cid()).Fields(mlog.Field("domain", domain)) + rlog := log.WithCid(mox.Cid()).With(slog.Any("domain", domain)) rlog.Info("sending dmarc report") if _, err := sendReportDomain(ctx, rlog, resolver, db, endTime, domain); err != nil { rlog.Errorx("sending dmarc aggregate report to domain", err) @@ -478,8 +479,8 @@ type recipient struct { maxSize uint64 } -func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) { - log = log.Fields(mlog.Field("uri", uri.Address)) +func parseRecipient(log mlog.Log, uri dmarc.URI) (r recipient, ok bool) { + log = log.With(slog.Any("uri", uri.Address)) u, err := url.Parse(uri.Address) if err != nil { @@ -510,14 +511,14 @@ func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) { r.maxSize *= 1024 * 1024 * 1024 * 1024 case "": default: - log.Debug("unrecognized max size unit in dmarc record rua value", mlog.Field("unit", uri.Unit)) + log.Debug("unrecognized max size unit in dmarc record rua value", slog.String("unit", uri.Unit)) return r, false } return r, true } -func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTime time.Time, domain string) { +func removeEvaluations(ctx context.Context, log mlog.Log, db *bstore.DB, endTime time.Time, domain string) { q := bstore.QueryDB[Evaluation](ctx, db) q.FilterLess("Evaluated", endTime) q.FilterNonzero(Evaluation{PolicyDomain: domain}) @@ -528,7 +529,7 @@ func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTim // replaceable for testing. var queueAdd = queue.Add -func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) { +func sendReportDomain(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) { dom, err := dns.ParseDomain(domain) if err != nil { return false, fmt.Errorf("parsing domain for sending reports: %v", err) @@ -567,7 +568,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, // evaluations regardless. We always use the latest DMARC record when sending, but // we'll lump all policies of the last interval into one report. // ../rfc/7489:1714 - status, _, record, _, _, err := dmarc.Lookup(ctx, resolver, dom) + status, _, record, _, _, err := dmarc.Lookup(ctx, log.Logger, resolver, dom) if err != nil { // todo future: we could perhaps still send this report, assuming the values we know. in case of temporary error, we could also schedule again regardless of next interval hour (we would now only retry a 24h-interval report after 24h passed). // Remove records unless it was a temporary error. We'll try again next round. @@ -588,8 +589,8 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, // Check if domain of rua recipient has the same organizational domain as for the // evaluations. If not, we need to verify we are allowed to send. - ruaOrgDom := publicsuffix.Lookup(ctx, r.address.Domain) - evalOrgDom := publicsuffix.Lookup(ctx, dom) + ruaOrgDom := publicsuffix.Lookup(ctx, log.Logger, r.address.Domain) + evalOrgDom := publicsuffix.Lookup(ctx, log.Logger, dom) if ruaOrgDom == evalOrgDom { recipients = append(recipients, r) @@ -599,12 +600,12 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, // Verify and follow addresses in other organizational domain through // ._report._dmarc. lookup. // ../rfc/7489:1556 - accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, resolver, evalOrgDom, r.address.Domain) + accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, log.Logger, resolver, evalOrgDom, r.address.Domain) log.Debugx("checking if rua address with different organization domain has opted into receiving dmarc reports", err, - mlog.Field("policydomain", evalOrgDom), - mlog.Field("destinationdomain", r.address.Domain), - mlog.Field("accepts", accepts), - mlog.Field("status", status)) + slog.Any("policydomain", evalOrgDom), + slog.Any("destinationdomain", r.address.Domain), + slog.Bool("accepts", accepts), + slog.Any("status", status)) if status == dmarc.StatusTemperror { // With a temporary error, we'll try to get the report the delivered anyway, // perhaps there are multiple recipients. @@ -626,7 +627,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, // alternative addresses and no new address specified). // ../rfc/7489:1600 foundReplacement := false - rlog := log.Fields(mlog.Field("followedaddress", uri.Address)) + rlog := log.With(slog.Any("followedaddress", uri.Address)) for _, record := range records { for _, exturi := range record.AggregateReportAddresses { extr, ok := parseRecipient(rlog, exturi) @@ -634,10 +635,10 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, continue } if extr.address.Domain != r.address.Domain { - rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", mlog.Field("externaladdress", extr.address)) + rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", slog.Any("externaladdress", extr.address)) errors = append(errors, fmt.Sprintf("rua %s is external domain with a replacement address %s with different host", r.address, extr.address)) } else { - rlog.Debug("using replacement rua address from external _report dmarc record", mlog.Field("externaladdress", extr.address)) + rlog.Debug("using replacement rua address from external _report dmarc record", slog.Any("externaladdress", extr.address)) foundReplacement = true recipients = append(recipients, extr) } @@ -744,7 +745,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, report.Records = append(report.Records, rc.ReportRecord) } - reportFile, err := store.CreateMessageTemp("dmarcreportout") + reportFile, err := store.CreateMessageTemp(log, "dmarcreportout") if err != nil { return false, fmt.Errorf("creating temporary file for outgoing dmarc aggregate report: %v", err) } @@ -767,7 +768,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, return true, fmt.Errorf("writing dmarc aggregate report as xml with gzip: %v", err) } - msgf, err := store.CreateMessageTemp("dmarcreportmsgout") + msgf, err := store.CreateMessageTemp(log, "dmarcreportmsgout") if err != nil { return false, fmt.Errorf("creating temporary message file with outgoing dmarc aggregate report: %v", err) } @@ -828,7 +829,7 @@ Period: %s - %s UTC return false, fmt.Errorf("querying suppress list: %v", err) } if exists { - log.Info("suppressing outgoing dmarc aggregate report", mlog.Field("reportingaddress", rcpt.address)) + log.Info("suppressing outgoing dmarc aggregate report", slog.Any("reportingaddress", rcpt.address)) continue } @@ -853,7 +854,7 @@ Period: %s - %s UTC log.Errorx("queueing message with dmarc aggregate report", err) metricReportError.Inc() } else { - log.Debug("dmarc aggregate report queued", mlog.Field("recipient", rcpt.address)) + log.Debug("dmarc aggregate report queued", slog.Any("recipient", rcpt.address)) queued = true metricReport.Inc() } @@ -873,7 +874,7 @@ Period: %s - %s UTC return true, nil } -func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { +func composeAggregateReport(ctx context.Context, log mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { xc := message.NewComposer(mf, 100*1024*1024) defer func() { x := recover() @@ -946,10 +947,10 @@ func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fro // Though this functionality is quite underspecified, we'll do our best to send our // an error report in case our report is too large for all recipients. // ../rfc/7489:1918 -func sendErrorReport(ctx context.Context, log *mlog.Log, db *bstore.DB, fromAddr smtp.Address, recipients []message.NameAddress, reportDomain dns.Domain, reportID string, reportMsgSize int64) error { +func sendErrorReport(ctx context.Context, log mlog.Log, db *bstore.DB, fromAddr smtp.Address, recipients []message.NameAddress, reportDomain dns.Domain, reportID string, reportMsgSize int64) error { log.Debug("no reporting addresses willing to accept report given size, queuing short error message") - msgf, err := store.CreateMessageTemp("dmarcreportmsg-out") + msgf, err := store.CreateMessageTemp(log, "dmarcreportmsg-out") if err != nil { return fmt.Errorf("creating temporary message file for outgoing dmarc error report: %v", err) } @@ -992,7 +993,7 @@ Submitting-URI: %s return fmt.Errorf("querying suppress list: %v", err) } if exists { - log.Info("suppressing outgoing dmarc error report", mlog.Field("reportingaddress", rcpt.Address)) + log.Info("suppressing outgoing dmarc error report", slog.Any("reportingaddress", rcpt.Address)) continue } @@ -1006,14 +1007,14 @@ Submitting-URI: %s log.Errorx("queueing message with dmarc error report", err) metricReportError.Inc() } else { - log.Debug("dmarc error report queued", mlog.Field("recipient", rcpt)) + log.Debug("dmarc error report queued", slog.Any("recipient", rcpt)) metricReport.Inc() } } return nil } -func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { +func composeErrorReport(ctx context.Context, log mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []message.NameAddress, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { xc := message.NewComposer(mf, 100*1024*1024) defer func() { x := recover() @@ -1058,7 +1059,7 @@ func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAdd return msgPrefix, xc.Has8bit, xc.SMTPUTF8, messageID, nil } -func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string { +func dkimSign(ctx context.Context, log mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string { // Add DKIM-Signature headers if we have a key for (a higher) domain than the from // address, which is a host name. A signature will only be useful with higher-level // domains if they have a relaxed dkim check (which is the default). If the dkim @@ -1068,7 +1069,7 @@ func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf for fd != zerodom { confDom, ok := mox.Conf.Domain(fd) if len(confDom.DKIM.Sign) > 0 { - dkimHeaders, err := dkim.Sign(ctx, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf) + dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf) if err != nil { log.Errorx("dkim-signing dmarc report, continuing without signature", err) metricReportError.Inc() diff --git a/dmarcdb/eval_test.go b/dmarcdb/eval_test.go index 35baaf9..04e5e9f 100644 --- a/dmarcdb/eval_test.go +++ b/dmarcdb/eval_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dmarcrpt" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" @@ -156,7 +158,7 @@ func TestEvaluations(t *testing.T) { } func TestSendReports(t *testing.T) { - mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) + mlog.SetConfig(map[string]slog.Level{"": slog.LevelDebug}) os.RemoveAll("../testdata/dmarcdb/data") mox.Context = ctxbg @@ -294,7 +296,7 @@ func TestSendReports(t *testing.T) { aggrAddrs := map[string]struct{}{} errorAddrs := map[string]struct{}{} - queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error { + queueAdd = func(ctx context.Context, log mlog.Log, qm *queue.Msg, msgFile *os.File) error { // Read message file. Also write copy to disk for inspection. buf, err := io.ReadAll(&moxio.AtReader{R: msgFile}) tcheckf(t, err, "read report message") @@ -309,7 +311,7 @@ func TestSendReports(t *testing.T) { } else { aggrAddrs[addr] = struct{}{} - feedback, err = dmarcrpt.ParseMessageReport(log, msgFile) + feedback, err = dmarcrpt.ParseMessageReport(log.Logger, msgFile) tcheckf(t, err, "parsing generated report message") } diff --git a/dmarcrpt/parse.go b/dmarcrpt/parse.go index 547b657..7660d58 100644 --- a/dmarcrpt/parse.go +++ b/dmarcrpt/parse.go @@ -12,6 +12,8 @@ import ( "net/http" "strings" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/message" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/moxio" @@ -34,9 +36,10 @@ func ParseReport(r io.Reader) (*Feedback, error) { // ParseMessageReport parses an aggregate feedback report from a mail message. The // maximum message size is 15MB, the maximum report size after decompression is // 20MB. -func ParseMessageReport(log *mlog.Log, r io.ReaderAt) (*Feedback, error) { +func ParseMessageReport(elog *slog.Logger, r io.ReaderAt) (*Feedback, error) { + log := mlog.New("dmarcrpt", elog) // ../rfc/7489:1801 - p, err := message.Parse(log, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024}) + p, err := message.Parse(log.Logger, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024}) if err != nil { return nil, fmt.Errorf("parsing mail message: %s", err) } @@ -44,7 +47,7 @@ func ParseMessageReport(log *mlog.Log, r io.ReaderAt) (*Feedback, error) { return parseMessageReport(log, p) } -func parseMessageReport(log *mlog.Log, p message.Part) (*Feedback, error) { +func parseMessageReport(log mlog.Log, p message.Part) (*Feedback, error) { // Pretty much any mime structure is allowed. ../rfc/7489:1861 // In practice, some parties will send the report as the only (non-multipart) // content of the message. @@ -54,7 +57,7 @@ func parseMessageReport(log *mlog.Log, p message.Part) (*Feedback, error) { } for { - sp, err := p.ParseNextPart(log) + sp, err := p.ParseNextPart(log.Logger) if err == io.EOF { return nil, ErrNoReport } diff --git a/dmarcrpt/parse_test.go b/dmarcrpt/parse_test.go index ade0134..64d2d36 100644 --- a/dmarcrpt/parse_test.go +++ b/dmarcrpt/parse_test.go @@ -11,7 +11,7 @@ import ( "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("dmarcrpt") +var pkglog = mlog.New("dmarcrpt", nil) const reportExample = ` @@ -137,7 +137,7 @@ func TestParseMessageReport(t *testing.T) { if err != nil { t.Fatalf("open %q: %s", p, err) } - _, err = ParseMessageReport(xlog, f) + _, err = ParseMessageReport(pkglog.Logger, f) if err != nil { t.Fatalf("ParseMessageReport: %q: %s", p, err) } @@ -145,7 +145,7 @@ func TestParseMessageReport(t *testing.T) { } // No report in a non-multipart message. - _, err = ParseMessageReport(xlog, strings.NewReader("From: \r\n\r\nNo report.\r\n")) + _, err = ParseMessageReport(pkglog.Logger, strings.NewReader("From: \r\n\r\nNo report.\r\n")) if err != ErrNoReport { t.Fatalf("message without report, got err %#v, expected ErrNoreport", err) } @@ -171,7 +171,7 @@ MIME-Version: 1.0 --===============5735553800636657282==-- `, "\n", "\r\n") - _, err = ParseMessageReport(xlog, strings.NewReader(multipartNoreport)) + _, err = ParseMessageReport(pkglog.Logger, strings.NewReader(multipartNoreport)) if err != ErrNoReport { t.Fatalf("message without report, got err %#v, expected ErrNoreport", err) } diff --git a/dns/resolver.go b/dns/resolver.go index 12ae375..9a1d100 100644 --- a/dns/resolver.go +++ b/dns/resolver.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -22,8 +24,6 @@ import ( // todo future: change to interface that is closer to DNS. 1. expose nxdomain vs success with zero entries: nxdomain means the name does not exist for any dns resource record type, success with zero records means the name exists for other types than the requested type; 2. add ability to not follow cname records when resolving. the net resolver automatically follows cnames for LookupHost, LookupIP, LookupIPAddr. when resolving names found in mx records, we explicitly must not follow cnames. that seems impossible at the moment. 3. when looking up a cname, actually lookup the record? "net" LookupCNAME will return the requested name with no error if there is no CNAME record. because it returns the canonical name. // todo future: add option to not use anything in the cache, for the admin pages where you check the latest DNS settings, ignoring old cached info. -var xlog = mlog.New("dns") - func init() { net.DefaultResolver.StrictErrors = true } @@ -74,6 +74,15 @@ func WithPackage(resolver Resolver, name string) Resolver { type StrictResolver struct { Pkg string // Name of subsystem that is making DNS requests, for metrics. Resolver *adns.Resolver // Where the actual lookups are done. If nil, adns.DefaultResolver is used for lookups. + Log *slog.Logger +} + +func (r StrictResolver) log() mlog.Log { + pkg := r.Pkg + if pkg == "" { + pkg = "dns" + } + return mlog.New(pkg, r.Log) } var _ Resolver = StrictResolver{} @@ -133,13 +142,12 @@ func (r StrictResolver) LookupPort(ctx context.Context, network, service string) start := time.Now() defer func() { metricLookupObserve(r.Pkg, "port", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "port"), - mlog.Field("network", network), - mlog.Field("service", service), - mlog.Field("resp", resp), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "port"), + slog.String("network", network), + slog.String("service", service), + slog.Int("resp", resp), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -152,13 +160,12 @@ func (r StrictResolver) LookupAddr(ctx context.Context, addr string) (resp []str start := time.Now() defer func() { metricLookupObserve(r.Pkg, "addr", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "addr"), - mlog.Field("addr", addr), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "addr"), + slog.String("addr", addr), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -179,13 +186,12 @@ func (r StrictResolver) LookupCNAME(ctx context.Context, host string) (resp stri start := time.Now() defer func() { metricLookupObserve(r.Pkg, "cname", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "cname"), - mlog.Field("host", host), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "cname"), + slog.String("host", host), + slog.String("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -209,13 +215,12 @@ func (r StrictResolver) LookupHost(ctx context.Context, host string) (resp []str start := time.Now() defer func() { metricLookupObserve(r.Pkg, "host", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "host"), - mlog.Field("host", host), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "host"), + slog.String("host", host), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -231,14 +236,13 @@ func (r StrictResolver) LookupIP(ctx context.Context, network, host string) (res start := time.Now() defer func() { metricLookupObserve(r.Pkg, "ip", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "ip"), - mlog.Field("network", network), - mlog.Field("host", host), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "ip"), + slog.String("network", network), + slog.String("host", host), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -254,13 +258,12 @@ func (r StrictResolver) LookupIPAddr(ctx context.Context, host string) (resp []n start := time.Now() defer func() { metricLookupObserve(r.Pkg, "ipaddr", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "ipaddr"), - mlog.Field("host", host), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "ipaddr"), + slog.String("host", host), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -276,13 +279,12 @@ func (r StrictResolver) LookupMX(ctx context.Context, name string) (resp []*net. start := time.Now() defer func() { metricLookupObserve(r.Pkg, "mx", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "mx"), - mlog.Field("name", name), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "mx"), + slog.String("name", name), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -298,13 +300,12 @@ func (r StrictResolver) LookupNS(ctx context.Context, name string) (resp []*net. start := time.Now() defer func() { metricLookupObserve(r.Pkg, "ns", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "ns"), - mlog.Field("name", name), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "ns"), + slog.String("name", name), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -320,16 +321,15 @@ func (r StrictResolver) LookupSRV(ctx context.Context, service, proto, name stri start := time.Now() defer func() { metricLookupObserve(r.Pkg, "srv", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "srv"), - mlog.Field("service", service), - mlog.Field("proto", proto), - mlog.Field("name", name), - mlog.Field("resp0", resp0), - mlog.Field("resp1", resp1), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "srv"), + slog.String("service", service), + slog.String("proto", proto), + slog.String("name", name), + slog.String("resp0", resp0), + slog.Any("resp1", resp1), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -345,13 +345,12 @@ func (r StrictResolver) LookupTXT(ctx context.Context, name string) (resp []stri start := time.Now() defer func() { metricLookupObserve(r.Pkg, "txt", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "txt"), - mlog.Field("name", name), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "txt"), + slog.String("name", name), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) @@ -367,15 +366,14 @@ func (r StrictResolver) LookupTLSA(ctx context.Context, port int, protocol, host start := time.Now() defer func() { metricLookupObserve(r.Pkg, "tlsa", err, start) - xlog.WithContext(ctx).Debugx("dns lookup result", err, - mlog.Field("pkg", r.Pkg), - mlog.Field("type", "tlsa"), - mlog.Field("port", port), - mlog.Field("protocol", protocol), - mlog.Field("host", host), - mlog.Field("resp", resp), - mlog.Field("authentic", result.Authentic), - mlog.Field("duration", time.Since(start)), + r.log().WithContext(ctx).Debugx("dns lookup result", err, + slog.String("type", "tlsa"), + slog.Int("port", port), + slog.String("protocol", protocol), + slog.String("host", host), + slog.Any("resp", resp), + slog.Bool("authentic", result.Authentic), + slog.Duration("duration", time.Since(start)), ) }() defer resolveErrorHint(&err) diff --git a/dnsbl/dnsbl.go b/dnsbl/dnsbl.go index 57dac58..afb66f8 100644 --- a/dnsbl/dnsbl.go +++ b/dnsbl/dnsbl.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -17,8 +19,6 @@ import ( "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("dnsbl") - var ( metricLookup = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -45,12 +45,12 @@ var ( ) // Lookup checks if "ip" occurs in the DNS block list "zone" (e.g. dnsbl.example.org). -func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) { - log := xlog.WithContext(ctx) +func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) { + log := mlog.New("dnsbl", elog) start := time.Now() defer func() { metricLookup.WithLabelValues(zone.Name(), string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("dnsbl lookup result", rerr, mlog.Field("zone", zone), mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start))) + log.Debugx("dnsbl lookup result", rerr, slog.Any("zone", zone), slog.Any("ip", ip), slog.Any("status", rstatus), slog.String("explanation", rexplanation), slog.Duration("duration", time.Since(start))) }() b := &strings.Builder{} @@ -93,7 +93,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net. if dns.IsNotFound(err) { return StatusFail, "", nil } else if err != nil { - log.Debugx("looking up txt record from dnsbl", err, mlog.Field("addr", addr)) + log.Debugx("looking up txt record from dnsbl", err, slog.String("addr", addr)) return StatusFail, "", nil } return StatusFail, strings.Join(txts, "; "), nil @@ -104,16 +104,16 @@ func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net. // Users of a DNSBL should periodically check if the DNSBL is still operating // properly. // For temporary errors, ErrDNS is returned. -func CheckHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rerr error) { - log := xlog.WithContext(ctx) +func CheckHealth(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone dns.Domain) (rerr error) { + log := mlog.New("dnsbl", elog) start := time.Now() defer func() { - log.Debugx("dnsbl healthcheck result", rerr, mlog.Field("zone", zone), mlog.Field("duration", time.Since(start))) + log.Debugx("dnsbl healthcheck result", rerr, slog.Any("zone", zone), slog.Duration("duration", time.Since(start))) }() // ../rfc/5782:355 - status1, _, err1 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 1)) - status2, _, err2 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 2)) + status1, _, err1 := Lookup(ctx, log.Logger, resolver, zone, net.IPv4(127, 0, 0, 1)) + status2, _, err2 := Lookup(ctx, log.Logger, resolver, zone, net.IPv4(127, 0, 0, 2)) if status1 == StatusPass && status2 == StatusFail { return nil } else if status1 == StatusFail { diff --git a/dnsbl/dnsbl_test.go b/dnsbl/dnsbl_test.go index 1d9f0ee..7a9d4ed 100644 --- a/dnsbl/dnsbl_test.go +++ b/dnsbl/dnsbl_test.go @@ -6,10 +6,12 @@ import ( "testing" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" ) func TestDNSBL(t *testing.T) { ctx := context.Background() + log := mlog.New("dnsbl", nil) resolver := dns.MockResolver{ A: map[string][]string{ @@ -23,7 +25,7 @@ func TestDNSBL(t *testing.T) { }, } - if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil { + if status, expl, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil { t.Fatalf("lookup: %v", err) } else if status != StatusFail { t.Fatalf("lookup, got status %v, expected fail", status) @@ -31,7 +33,7 @@ func TestDNSBL(t *testing.T) { t.Fatalf("lookup, got explanation %q", expl) } - if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil { + if status, expl, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil { t.Fatalf("lookup: %v", err) } else if status != StatusFail { t.Fatalf("lookup, got status %v, expected fail", status) @@ -39,17 +41,17 @@ func TestDNSBL(t *testing.T) { t.Fatalf("lookup, got explanation %q", expl) } - if status, _, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil { + if status, _, err := Lookup(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil { t.Fatalf("lookup: %v", err) } else if status != StatusPass { t.Fatalf("lookup, got status %v, expected pass", status) } // ../rfc/5782:357 - if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.com"}); err != nil { + if err := CheckHealth(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.com"}); err != nil { t.Fatalf("dnsbl not healthy: %v", err) } - if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.org"}); err == nil { + if err := CheckHealth(ctx, log.Logger, resolver, dns.Domain{ASCII: "example.org"}); err == nil { t.Fatalf("bad dnsbl is healthy") } @@ -58,7 +60,7 @@ func TestDNSBL(t *testing.T) { "1.0.0.127.example.com.": {"127.0.0.2"}, // Should not be present in healthy dnsbl. }, } - if err := CheckHealth(ctx, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil { + if err := CheckHealth(ctx, log.Logger, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil { t.Fatalf("bad dnsbl is healthy") } } diff --git a/dsn/dsn.go b/dsn/dsn.go index a9147ae..2e6605b 100644 --- a/dsn/dsn.go +++ b/dsn/dsn.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/message" @@ -135,7 +137,7 @@ type Recipient struct { // DSN. // // DKIM signatures are added if DKIM signing is configured for the "from" domain. -func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) { +func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) { // ../rfc/3462:119 // ../rfc/3464:377 // We'll make a multipart/report with 2 or 3 parts: @@ -381,9 +383,9 @@ func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) { continue } - dkimHeaders, err := dkim.Sign(context.Background(), m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data)) + dkimHeaders, err := dkim.Sign(context.Background(), log.Logger, m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data)) if err != nil { - log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, mlog.Field("domain", fd)) + log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, slog.Any("domain", fd)) } else { data = append([]byte(dkimHeaders), data...) } diff --git a/dsn/dsn_test.go b/dsn/dsn_test.go index fc60ce1..c3edc3f 100644 --- a/dsn/dsn_test.go +++ b/dsn/dsn_test.go @@ -20,7 +20,7 @@ import ( "github.com/mjl-/mox/smtp" ) -var xlog = mlog.New("dsn") +var pkglog = mlog.New("dsn", nil) func xparseDomain(s string) dns.Domain { d, err := dns.ParseDomain(s) @@ -36,7 +36,7 @@ func xparseIPDomain(s string) dns.IPDomain { func tparseMessage(t *testing.T, data []byte, nparts int) (*Message, *message.Part) { t.Helper() - m, p, err := Parse(xlog, bytes.NewReader(data)) + m, p, err := Parse(pkglog.Logger, bytes.NewReader(data)) if err != nil { t.Fatalf("parsing dsn: %v", err) } @@ -75,7 +75,7 @@ func tcompareReader(t *testing.T, r io.Reader, exp []byte) { } func TestDSN(t *testing.T) { - log := mlog.New("dsn") + log := mlog.New("dsn", nil) now := time.Now() @@ -143,7 +143,7 @@ func TestDSN(t *testing.T) { "testsel._domainkey.mox.example.": {"v=DKIM1;h=sha256;t=s;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3ZId3ys70VFspp/VMFaxMOrNjHNPg04NOE1iShih16b3Ex7hHBOgC1UvTGSmrMlbCB1OxTXkvf6jW6S4oYRnZYVNygH6zKUwYYhaSaGIg1xA/fDn+IgcTRyLoXizMUgUgpTGyxhNrwIIWv+i7jjbs3TKpP3NU4owQ/rxowmSNqg+fHIF1likSvXvljYS" + "jaFXXnWfYibW7TdDCFFpN4sB5o13+as0u4vLw6MvOi59B1tLype1LcHpi1b9PfxNtznTTdet3kL0paxIcWtKHT0LDPUos8YYmiPa5nGbUqlC7d+4YT2jQPvwGxCws1oo2Tw6nj1UaihneYGAyvEky49FBwIDAQAB"}, }, } - results, err := dkim.Verify(context.Background(), resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false) + results, err := dkim.Verify(context.Background(), log.Logger, resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false) if err != nil { t.Fatalf("dkim verify: %v", err) } diff --git a/dsn/parse.go b/dsn/parse.go index 161508b..f5bcf99 100644 --- a/dsn/parse.go +++ b/dsn/parse.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/message" "github.com/mjl-/mox/mlog" @@ -23,17 +25,19 @@ import ( // The first return value is the machine-parsed DSN message. The second value is // the entire MIME multipart message. Use its Parts field to access the // human-readable text and optional original message/headers. -func Parse(log *mlog.Log, r io.ReaderAt) (*Message, *message.Part, error) { +func Parse(elog *slog.Logger, r io.ReaderAt) (*Message, *message.Part, error) { + log := mlog.New("dsn", elog) + // DSNs can mix and match subtypes with and without utf-8. ../rfc/6533:441 - part, err := message.Parse(log, false, r) + part, err := message.Parse(log.Logger, false, r) if err != nil { return nil, nil, fmt.Errorf("parsing message: %v", err) } if part.MediaType != "MULTIPART" || part.MediaSubType != "REPORT" { return nil, nil, fmt.Errorf(`message has content-type %q, must have "message/report"`, strings.ToLower(part.MediaType+"/"+part.MediaSubType)) } - err = part.Walk(log, nil) + err = part.Walk(log.Logger, nil) if err != nil { return nil, nil, fmt.Errorf("parsing message parts: %v", err) } diff --git a/export.go b/export.go index 3f331a4..0368a2c 100644 --- a/export.go +++ b/export.go @@ -8,7 +8,6 @@ import ( "github.com/mjl-/bstore" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/store" ) @@ -66,7 +65,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) { }() a := store.DirArchiver{Dir: dst} - err = store.ExportMessages(context.Background(), mlog.New("export"), db, accountDir, a, !mbox, mailbox) + err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox) xcheckf(err, "exporting messages") err = a.Close() xcheckf(err, "closing archiver") diff --git a/gentestdata.go b/gentestdata.go index cec8561..6e8e5bc 100644 --- a/gentestdata.go +++ b/gentestdata.go @@ -54,7 +54,6 @@ func cmdGentestdata(c *cmd) { return f } - log := mlog.New("gentestdata") ctxbg := context.Background() mox.Conf.Log[""] = mlog.LevelInfo mlog.SetConfig(mox.Conf.Log) @@ -217,7 +216,7 @@ Accounts: xcheckf(err, "tlsrptdb init") tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport)) xcheckf(err, "parsing tls report") - err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr) + err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr) xcheckf(err, "adding tls report") // Populate queue, with a message. @@ -234,22 +233,22 @@ Accounts: _, err = fmt.Fprint(mf, qmsg) xcheckf(err, "writing message") qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "", prefix, nil) - err = queue.Add(ctxbg, log, &qm, mf) + err = queue.Add(ctxbg, c.log, &qm, mf) xcheckf(err, "enqueue message") // Create three accounts. // First account without messages. - accTest0, err := store.OpenAccount("test0") + accTest0, err := store.OpenAccount(c.log, "test0") xcheckf(err, "open account test0") - err = accTest0.ThreadingWait(log) + err = accTest0.ThreadingWait(c.log) xcheckf(err, "wait for threading to finish") err = accTest0.Close() xcheckf(err, "close account") // Second account with one message. - accTest1, err := store.OpenAccount("test1") + accTest1, err := store.OpenAccount(c.log, "test1") xcheckf(err, "open account test1") - err = accTest1.ThreadingWait(log) + err = accTest1.ThreadingWait(c.log) xcheckf(err, "wait for threading to finish") err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error { inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() @@ -285,7 +284,7 @@ Accounts: xcheckf(err, "creating temp file for delivery") _, err = fmt.Fprint(mf, msg) xcheckf(err, "writing deliver message to file") - err = accTest1.DeliverMessage(log, tx, &m, mf, false, true, false) + err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false) mfname := mf.Name() xcheckf(err, "add message to account test1") @@ -307,9 +306,9 @@ Accounts: xcheckf(err, "close account") // Third account with two messages and junkfilter. - accTest2, err := store.OpenAccount("test2") + accTest2, err := store.OpenAccount(c.log, "test2") xcheckf(err, "open account test2") - err = accTest2.ThreadingWait(log) + err = accTest2.ThreadingWait(c.log) xcheckf(err, "wait for threading to finish") err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error { inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() @@ -345,7 +344,7 @@ Accounts: xcheckf(err, "creating temp file for delivery") _, err = fmt.Fprint(mf0, msg0) xcheckf(err, "writing deliver message to file") - err = accTest2.DeliverMessage(log, tx, &m0, mf0, false, false, false) + err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false) xcheckf(err, "add message to account test2") mf0name := mf0.Name() @@ -376,7 +375,7 @@ Accounts: xcheckf(err, "creating temp file for delivery") _, err = fmt.Fprint(mf1, msg1) xcheckf(err, "writing deliver message to file") - err = accTest2.DeliverMessage(log, tx, &m1, mf1, false, false, false) + err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false) xcheckf(err, "add message to account test2") mf1name := mf1.Name() diff --git a/http/autoconf.go b/http/autoconf.go index 3133891..09a3690 100644 --- a/http/autoconf.go +++ b/http/autoconf.go @@ -6,11 +6,12 @@ import ( "net/http" "strings" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "rsc.io/qr" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/smtp" ) @@ -55,7 +56,7 @@ var ( // User should create a DNS record: autoconfig. (CNAME or A). // See https://wiki.mozilla.org/Thunderbird:Autoconfiguration:ConfigFileFormat func autoconfHandle(w http.ResponseWriter, r *http.Request) { - log := xlog.WithContext(r.Context()) + log := pkglog.WithContext(r.Context()) var addrDom string defer func() { @@ -63,7 +64,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) { }() email := r.FormValue("emailaddress") - log.Debug("autoconfig request", mlog.Field("email", email)) + log.Debug("autoconfig request", slog.String("email", email)) addr, err := smtp.ParseAddress(email) if err != nil { http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest) @@ -143,7 +144,7 @@ func autoconfHandle(w http.ResponseWriter, r *http.Request) { // // Thunderbird does understand autodiscover. func autodiscoverHandle(w http.ResponseWriter, r *http.Request) { - log := xlog.WithContext(r.Context()) + log := pkglog.WithContext(r.Context()) var addrDom string defer func() { @@ -161,7 +162,7 @@ func autodiscoverHandle(w http.ResponseWriter, r *http.Request) { return } - log.Debug("autodiscover request", mlog.Field("email", req.Request.EmailAddress)) + log.Debug("autodiscover request", slog.String("email", req.Request.EmailAddress)) addr, err := smtp.ParseAddress(req.Request.EmailAddress) if err != nil { diff --git a/http/gzcache.go b/http/gzcache.go index 2bc10ce..e910cb6 100644 --- a/http/gzcache.go +++ b/http/gzcache.go @@ -16,6 +16,8 @@ import ( "sync" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/mlog" ) @@ -75,7 +77,7 @@ func loadStaticGzipCache(dir string, maxSize int64) { os.MkdirAll(dir, 0700) entries, err := os.ReadDir(dir) if err != nil && !os.IsNotExist(err) { - xlog.Errorx("listing static gzip cache files", err, mlog.Field("dir", dir)) + pkglog.Errorx("listing static gzip cache files", err, slog.String("dir", dir)) } for _, e := range entries { name := e.Name() @@ -111,9 +113,9 @@ func loadStaticGzipCache(dir string, maxSize int64) { atime, err = statAtime(fi.Sys()) } if err != nil { - xlog.Infox("removing unusable/unrecognized file in static gzip cache dir", err) + pkglog.Infox("removing unusable/unrecognized file in static gzip cache dir", err) xerr := os.Remove(filepath.Join(dir, name)) - xlog.Check(xerr, "removing unusable file in static gzip cache dir", mlog.Field("error", err), mlog.Field("dir", dir), mlog.Field("filename", name)) + pkglog.Check(xerr, "removing unusable file in static gzip cache dir", slog.Any("error", err), slog.String("dir", dir), slog.String("filename", name)) continue } staticgzcache.paths[path] = gzfile{ @@ -163,7 +165,7 @@ func (c *gzcache) evictPath(path string) { c.unlink(gf.use) c.size -= gf.gzsize err := os.Remove(staticCachePath(c.dir, path, gf.mtime)) - xlog.Check(err, "removing cached gzipped static file", mlog.Field("path", path)) + pkglog.Check(err, "removing cached gzipped static file", slog.String("path", path)) } // Open cached file for path, requiring it has mtime. If there is no usable cached @@ -189,7 +191,7 @@ func (c *gzcache) openPath(path string, mtime int64) (*os.File, int64) { p := staticCachePath(c.dir, path, gf.mtime) f, err := os.Open(p) if err != nil { - xlog.Errorx("open static cached gzip file, removing from cache", err, mlog.Field("path", path)) + pkglog.Errorx("open static cached gzip file, removing from cache", err, slog.String("path", path)) // Perhaps someone removed the file? Remove from cache, it will be recreated. c.evictPath(path) return nil, 0 @@ -303,8 +305,8 @@ type staticgzcacheReplacer struct { handled bool } -func (w *staticgzcacheReplacer) logger() *mlog.Log { - return xlog.WithContext(w.r.Context()) +func (w *staticgzcacheReplacer) logger() mlog.Log { + return pkglog.WithContext(w.r.Context()) } // Header returns the header of the underlying ResponseWriter. @@ -353,7 +355,7 @@ func (w *staticgzcacheReplacer) WriteHeader(statusCode int) { p := staticCachePath(staticgzcache.dir, w.uncomprPath, w.uncomprMtime.UnixNano()) ngzf, err := os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) if err != nil { - w.logger().Errorx("create new static gzip cache file", err, mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p)) + w.logger().Errorx("create new static gzip cache file", err, slog.String("requestpath", w.uncomprPath), slog.String("fspath", p)) staticgzcache.abortPath(w.uncomprPath) return } @@ -361,9 +363,9 @@ func (w *staticgzcacheReplacer) WriteHeader(statusCode int) { if ngzf != nil { staticgzcache.abortPath(w.uncomprPath) err := ngzf.Close() - w.logger().Check(err, "closing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p)) + w.logger().Check(err, "closing failed static gzip cache file", slog.String("requestpath", w.uncomprPath), slog.String("fspath", p)) err = os.Remove(p) - w.logger().Check(err, "removing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p)) + w.logger().Check(err, "removing failed static gzip cache file", slog.String("requestpath", w.uncomprPath), slog.String("fspath", p)) } }() diff --git a/http/mtasts.go b/http/mtasts.go index 90d7bc8..47f90eb 100644 --- a/http/mtasts.go +++ b/http/mtasts.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" @@ -13,8 +15,8 @@ import ( ) func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) { - log := func() *mlog.Log { - return xlog.WithContext(r.Context()) + log := func() mlog.Log { + return pkglog.WithContext(r.Context()) } host := strings.ToLower(r.Host) @@ -30,7 +32,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) { } domain, err := dns.ParseDomain(host) if err != nil { - log().Errorx("mtasts policy request: bad domain", err, mlog.Field("host", host)) + log().Errorx("mtasts policy request: bad domain", err, slog.String("host", host)) http.NotFound(w, r) return } @@ -51,7 +53,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) { } d, err := dns.ParseDomain(s) if err != nil { - log().Errorx("bad domain in mtasts config", err, mlog.Field("domain", s)) + log().Errorx("bad domain in mtasts config", err, slog.String("domain", s)) http.Error(w, "500 - internal server error - invalid domain in configuration", http.StatusInternalServerError) return } diff --git a/http/web.go b/http/web.go index 5ced90e..6b1e460 100644 --- a/http/web.go +++ b/http/web.go @@ -21,6 +21,7 @@ import ( _ "net/http/pprof" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -37,7 +38,7 @@ import ( "github.com/mjl-/mox/webmail" ) -var xlog = mlog.New("http") +var pkglog = mlog.New("http", nil) var ( // metricRequest tracks performance (time to write response header) of server. @@ -96,11 +97,11 @@ type loggingWriter struct { Err error WebsocketResponse bool // If this was a successful websocket connection with backend. SizeFromClient, SizeToClient int64 // Websocket data. - Fields []mlog.Pair // Additional fields to log. + Attrs []slog.Attr // Additional fields to log. } -func (w *loggingWriter) AddField(p mlog.Pair) { - w.Fields = append(w.Fields, p) +func (w *loggingWriter) AddAttr(a slog.Attr) { + w.Attrs = append(w.Attrs, a) } func (w *loggingWriter) Flush() { @@ -310,43 +311,43 @@ func (w *loggingWriter) Done() { if err == nil { err = w.R.Context().Err() } - fields := []mlog.Pair{ - mlog.Field("httpaccess", ""), - mlog.Field("handler", w.Handler), - mlog.Field("method", method), - mlog.Field("url", w.R.URL), - mlog.Field("host", w.R.Host), - mlog.Field("duration", time.Since(w.Start)), - mlog.Field("statuscode", w.StatusCode), - mlog.Field("proto", strings.ToLower(w.R.Proto)), - mlog.Field("remoteaddr", w.R.RemoteAddr), - mlog.Field("tlsinfo", tlsinfo), - mlog.Field("useragent", w.R.Header.Get("User-Agent")), - mlog.Field("referrr", w.R.Header.Get("Referrer")), + attrs := []slog.Attr{ + slog.String("httpaccess", ""), + slog.String("handler", w.Handler), + slog.String("method", method), + slog.Any("url", w.R.URL), + slog.String("host", w.R.Host), + slog.Duration("duration", time.Since(w.Start)), + slog.Int("statuscode", w.StatusCode), + slog.String("proto", strings.ToLower(w.R.Proto)), + slog.Any("remoteaddr", w.R.RemoteAddr), + slog.String("tlsinfo", tlsinfo), + slog.String("useragent", w.R.Header.Get("User-Agent")), + slog.String("referrr", w.R.Header.Get("Referrer")), } if w.WebsocketRequest { - fields = append(fields, - mlog.Field("websocketrequest", true), + attrs = append(attrs, + slog.Bool("websocketrequest", true), ) } if w.WebsocketResponse { - fields = append(fields, - mlog.Field("websocket", true), - mlog.Field("sizetoclient", w.SizeToClient), - mlog.Field("sizefromclient", w.SizeFromClient), + attrs = append(attrs, + slog.Bool("websocket", true), + slog.Int64("sizetoclient", w.SizeToClient), + slog.Int64("sizefromclient", w.SizeFromClient), ) } else if w.UncompressedSize > 0 { - fields = append(fields, - mlog.Field("size", w.Size), - mlog.Field("uncompressedsize", w.UncompressedSize), + attrs = append(attrs, + slog.Int64("size", w.Size), + slog.Int64("uncompressedsize", w.UncompressedSize), ) } else { - fields = append(fields, - mlog.Field("size", w.Size), + attrs = append(attrs, + slog.Int64("size", w.Size), ) } - fields = append(fields, w.Fields...) - xlog.WithContext(w.R.Context()).Debugx("http request", err, fields...) + attrs = append(attrs, w.Attrs...) + pkglog.WithContext(w.R.Context()).Debugx("http request", err, attrs...) } // Set some http headers that should prevent potential abuse. Better safe than sorry. @@ -405,9 +406,9 @@ func (s *serve) ServeHTTP(xw http.ResponseWriter, r *http.Request) { // Rate limiting as early as possible. ipstr, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - xlog.Debugx("split host:port client remoteaddr", err, mlog.Field("remoteaddr", r.RemoteAddr)) + pkglog.Debugx("split host:port client remoteaddr", err, slog.Any("remoteaddr", r.RemoteAddr)) } else if ip := net.ParseIP(ipstr); ip == nil { - xlog.Debug("parsing ip for client remoteaddr", mlog.Field("remoteaddr", r.RemoteAddr)) + pkglog.Debug("parsing ip for client remoteaddr", slog.Any("remoteaddr", r.RemoteAddr)) } else if !limiterConnectionrate.Add(ip, now, 1) { method := metricHTTPMethod(r.Method) proto := "http" @@ -649,7 +650,7 @@ func Listen() { // Importing net/http/pprof registers handlers on the default serve mux. port := config.Port(l.PprofHTTP.Port, 8011) if _, ok := portServe[port]; ok { - xlog.Fatal("cannot serve pprof on same endpoint as other http services") + pkglog.Fatal("cannot serve pprof on same endpoint as other http services") } srv := &serve{[]string{"pprof-http"}, nil, nil, false} portServe[port] = srv @@ -686,7 +687,7 @@ func Listen() { // presence of TLS certificates for. for _, name := range mox.Conf.Domains() { if dom, err := dns.ParseDomain(name); err != nil { - xlog.Errorx("parsing domain from config", err) + pkglog.Errorx("parsing domain from config", err) } else if d, _ := mox.Conf.Domain(dom); d.DMARC != nil && d.DMARC.Domain != "" && d.DMARC.DNSDomain != dom { // Do not gather autoconfig name if this domain is configured to process reports // for domains hosted elsewhere. @@ -695,7 +696,7 @@ func Listen() { autoconfdom, err := dns.ParseDomain("autoconfig." + name) if err != nil { - xlog.Errorx("parsing domain from config for autoconfig", err) + pkglog.Errorx("parsing domain from config for autoconfig", err) } else { hosts[autoconfdom] = struct{}{} } @@ -745,20 +746,20 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st if tlsConfig == nil { protocol = "http" if os.Getuid() == 0 { - xlog.Print("http listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr)) + pkglog.Print("http listener", slog.String("name", name), slog.String("kinds", strings.Join(kinds, ",")), slog.String("address", addr)) } ln, err = mox.Listen(mox.Network(ip), addr) if err != nil { - xlog.Fatalx("http: listen", err, mlog.Field("addr", addr)) + pkglog.Fatalx("http: listen", err, slog.Any("addr", addr)) } } else { protocol = "https" if os.Getuid() == 0 { - xlog.Print("https listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr)) + pkglog.Print("https listener", slog.String("name", name), slog.String("kinds", strings.Join(kinds, ",")), slog.String("address", addr)) } ln, err = mox.Listen(mox.Network(ip), addr) if err != nil { - xlog.Fatalx("https: listen", err, mlog.Field("addr", addr)) + pkglog.Fatalx("https: listen", err, slog.String("addr", addr)) } ln = tls.NewListener(ln, tlsConfig) } @@ -768,11 +769,11 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st TLSConfig: tlsConfig, ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 65 * time.Second, // Chrome closes connections after 60 seconds, firefox after 115 seconds. - ErrorLog: golog.New(mlog.ErrWriter(xlog.Fields(mlog.Field("pkg", "net/http")), mlog.LevelInfo, protocol+" error"), "", 0), + ErrorLog: golog.New(mlog.LogWriter(pkglog.With(slog.String("pkg", "net/http")), slog.LevelInfo, protocol+" error"), "", 0), } serve := func() { err := server.Serve(ln) - xlog.Fatalx(protocol+": serve", err) + pkglog.Fatalx(protocol+": serve", err) } servers = append(servers, serve) } @@ -815,9 +816,9 @@ func Serve() { SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}, SupportedVersions: []uint16{tls.VersionTLS13}, } - xlog.Print("ensuring certificate availability", mlog.Field("hostname", host)) + pkglog.Print("ensuring certificate availability", slog.Any("hostname", host)) if _, err := m.Manager.GetCertificate(hello); err != nil { - xlog.Errorx("requesting automatic certificate", err, mlog.Field("hostname", host)) + pkglog.Errorx("requesting automatic certificate", err, slog.Any("hostname", host)) } } } diff --git a/http/webserver.go b/http/webserver.go index 611e4ff..ec92679 100644 --- a/http/webserver.go +++ b/http/webserver.go @@ -25,6 +25,8 @@ import ( "syscall" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/config" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" @@ -149,8 +151,8 @@ table > tbody > tr:nth-child(odd) { background-color: #f8f8f8; } // file is returned. Otherwise, for directories with ListFiles configured, a // directory listing is returned. func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *http.Request) (handled bool) { - log := func() *mlog.Log { - return xlog.WithContext(r.Context()) + log := func() mlog.Log { + return pkglog.WithContext(r.Context()) } if r.Method != "GET" && r.Method != "HEAD" { if h.ContinueNotFound { @@ -217,7 +219,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r * var ifi os.FileInfo ifi, err = index.Stat() if err != nil { - log().Errorx("stat index.html in directory we cannot list", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) + log().Errorx("stat index.html in directory we cannot list", err, slog.Any("url", r.URL), slog.String("fspath", fspath)) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) return true } @@ -228,7 +230,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r * http.Error(w, "403 - permission denied", http.StatusForbidden) return true } - log().Errorx("open file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) + log().Errorx("open file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath)) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) return true } @@ -236,7 +238,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r * fi, err := f.Stat() if err != nil { - log().Errorx("stat file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) + log().Errorx("stat file for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath)) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) return true } @@ -274,7 +276,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r * } } if !os.IsNotExist(err) { - log().Errorx("stat for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) + log().Errorx("stat for static file serving", err, slog.Any("url", r.URL), slog.String("fspath", fspath)) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) return true } @@ -315,7 +317,7 @@ func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r * if err == io.EOF { break } else if err != nil { - log().Errorx("reading directory for file listing", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath)) + log().Errorx("reading directory for file listing", err, slog.Any("url", r.URL), slog.String("fspath", fspath)) http.Error(w, "500 - internal server error"+recvid(r), http.StatusInternalServerError) return true } @@ -398,8 +400,8 @@ func HandleRedirect(h *config.WebRedirect, w http.ResponseWriter, r *http.Reques // connections by monitoring the websocket handshake and then just passing along the // websocket frames. func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) { - log := func() *mlog.Log { - return xlog.WithContext(r.Context()) + log := func() mlog.Log { + return pkglog.WithContext(r.Context()) } xr := *r @@ -459,13 +461,13 @@ func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request, // ReverseProxy will append any remaining path to the configured target URL. proxy := httputil.NewSingleHostReverseProxy(h.TargetURL) proxy.FlushInterval = time.Duration(-1) // Flush after each write. - proxy.ErrorLog = golog.New(mlog.ErrWriter(mlog.New("net/http/httputil").WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0) + proxy.ErrorLog = golog.New(mlog.LogWriter(mlog.New("net/http/httputil", nil).WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0) proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { if errors.Is(err, context.Canceled) { - log().Debugx("forwarding request to backend webserver", err, mlog.Field("url", r.URL)) + log().Debugx("forwarding request to backend webserver", err, slog.Any("url", r.URL)) return } - log().Errorx("forwarding request to backend webserver", err, mlog.Field("url", r.URL)) + log().Errorx("forwarding request to backend webserver", err, slog.Any("url", r.URL)) if os.IsTimeout(err) { http.Error(w, "504 - gateway timeout"+recvid(r), http.StatusGatewayTimeout) } else { @@ -493,8 +495,8 @@ var errNotImplemented = errors.New("functionality not yet implemented") // work for little benefit. Besides, the whole point of websockets is to exchange // bytes without HTTP being in the way, so let's do that. func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) { - log := func() *mlog.Log { - return xlog.WithContext(r.Context()) + log := func() mlog.Log { + return pkglog.WithContext(r.Context()) } lw := w.(*loggingWriter) @@ -658,8 +660,8 @@ func forwardWebsocket(h *config.WebForward, w http.ResponseWriter, r *http.Reque } func websocketTransact(ctx context.Context, targetURL *url.URL, r *http.Request) (rresp *http.Response, rconn net.Conn, rerr error) { - log := func() *mlog.Log { - return xlog.WithContext(r.Context()) + log := func() mlog.Log { + return pkglog.WithContext(r.Context()) } // Dial the backend, possibly doing TLS. We assume the net/http DefaultTransport is diff --git a/imapserver/fetch.go b/imapserver/fetch.go index 9b55d76..5541fda 100644 --- a/imapserver/fetch.go +++ b/imapserver/fetch.go @@ -12,11 +12,11 @@ import ( "strings" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/mjl-/bstore" "github.com/mjl-/mox/message" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/store" @@ -233,7 +233,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { for _, uid := range uids { cmd.uid = uid - mlog.Field("processing uid", mlog.Field("uid", uid)) + cmd.conn.log.Debug("processing uid", slog.Any("uid", uid)) cmd.process(atts) } @@ -326,7 +326,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { cmd.expungeIssued = true return } - cmd.conn.log.Infox("processing fetch attribute", err, mlog.Field("uid", cmd.uid)) + cmd.conn.log.Infox("processing fetch attribute", err, slog.Any("uid", cmd.uid)) xuserErrorf("processing fetch attribute: %v", err) }() diff --git a/imapserver/fuzz_test.go b/imapserver/fuzz_test.go index da8ce4f..997be5e 100644 --- a/imapserver/fuzz_test.go +++ b/imapserver/fuzz_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/mjl-/mox/imapclient" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) @@ -58,17 +59,18 @@ func FuzzServer(f *testing.F) { f.Add(tag + cmd) } + log := mlog.New("imapserver", nil) mox.Context = ctxbg mox.ConfigStaticPath = filepath.FromSlash("../testdata/imapserverfuzz/mox.conf") mox.MustLoadConfig(true, false) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) os.RemoveAll(dataDir) - acc, err := store.OpenAccount("mjl") + acc, err := store.OpenAccount(log, "mjl") if err != nil { f.Fatalf("open account: %v", err) } defer acc.Close() - err = acc.SetPassword("testtest") + err = acc.SetPassword(log, "testtest") if err != nil { f.Fatalf("set password: %v", err) } diff --git a/imapserver/parse.go b/imapserver/parse.go index 0dc0a40..8008326 100644 --- a/imapserver/parse.go +++ b/imapserver/parse.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/mjl-/mox/mlog" + "golang.org/x/exp/slog" ) var ( @@ -402,7 +402,7 @@ func (p *parser) xmailbox() string { if !p.conn.enabled[capIMAP4rev2] { ns, err := utf7decode(s) if err != nil { - p.conn.log.Infox("decoding utf7 or mailbox name", err, mlog.Field("name", s)) + p.conn.log.Infox("decoding utf7 or mailbox name", err, slog.String("name", s)) } else { s = ns } diff --git a/imapserver/search.go b/imapserver/search.go index 63448c6..b1bfc0b 100644 --- a/imapserver/search.go +++ b/imapserver/search.go @@ -5,10 +5,11 @@ import ( "net/textproto" "strings" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/message" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/store" ) @@ -393,7 +394,7 @@ func (s *search) match0(sk searchKey) bool { lower := strings.ToLower(value) h, err := s.p.Header() if err != nil { - c.log.Debugx("parsing message header", err, mlog.Field("uid", s.uid)) + c.log.Debugx("parsing message header", err, slog.Any("uid", s.uid)) return false } for _, v := range h.Values(field) { @@ -517,7 +518,7 @@ func (s *search) match0(sk searchKey) bool { } if s.p == nil { - c.log.Info("missing parsed message, not matching", mlog.Field("uid", s.uid)) + c.log.Info("missing parsed message, not matching", slog.Any("uid", s.uid)) return false } @@ -546,7 +547,7 @@ func (s *search) match0(sk searchKey) bool { lower := strings.ToLower(sk.astring) h, err := s.p.Header() if err != nil { - c.log.Errorx("parsing header for search", err, mlog.Field("uid", s.uid)) + c.log.Errorx("parsing header for search", err, slog.Any("uid", s.uid)) return false } k := textproto.CanonicalMIMEHeaderKey(sk.headerField) diff --git a/imapserver/server.go b/imapserver/server.go index 9e1735c..599b570 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -56,10 +56,12 @@ import ( "runtime/debug" "sort" "strings" + "sync" "time" "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -78,10 +80,6 @@ import ( "github.com/mjl-/mox/store" ) -// Most logging should be done through conn.log* functions. -// Only use imaplog in contexts without connection. -var xlog = mlog.New("imapserver") - var ( metricIMAPConnection = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -180,7 +178,7 @@ type conn struct { cmdMetric string // Currently executing, for metrics. cmdStart time.Time ncmds int // Number of commands processed. Used to abort connection when first incoming command is unknown/invalid. - log *mlog.Log + log mlog.Log enabled map[capability]bool // All upper-case. // Set by SEARCH with SAVE. Can be used by commands accepting a sequence-set with @@ -338,14 +336,15 @@ func Listen() { var servers []func() func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, xtls, noRequireSTARTTLS bool) { + log := mlog.New("imapserver", nil) addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) if os.Getuid() == 0 { - xlog.Print("listening for imap", mlog.Field("listener", listenerName), mlog.Field("addr", addr), mlog.Field("protocol", protocol)) + log.Print("listening for imap", slog.String("listener", listenerName), slog.String("addr", addr), slog.String("protocol", protocol)) } network := mox.Network(ip) ln, err := mox.Listen(network, addr) if err != nil { - xlog.Fatalx("imap: listen for imap", err, mlog.Field("protocol", protocol), mlog.Field("listener", listenerName)) + log.Fatalx("imap: listen for imap", err, slog.String("protocol", protocol), slog.String("listener", listenerName)) } if xtls { ln = tls.NewListener(ln, tlsConfig) @@ -355,7 +354,7 @@ func listen1(protocol, listenerName, ip string, port int, tlsConfig *tls.Config, for { conn, err := ln.Accept() if err != nil { - xlog.Infox("imap: accept", err, mlog.Field("protocol", protocol), mlog.Field("listener", listenerName)) + log.Infox("imap: accept", err, slog.String("protocol", protocol), slog.String("listener", listenerName)) continue } @@ -441,7 +440,7 @@ func (c *conn) Write(buf []byte) (int, error) { return n, nil } -func (c *conn) xtrace(level mlog.Level) func() { +func (c *conn) xtrace(level slog.Level) func() { c.xflush() c.tr.SetTrace(level) c.tw.SetTrace(level) @@ -469,7 +468,7 @@ func (c *conn) readline0() (string, error) { err := c.conn.SetReadDeadline(time.Now().Add(d)) c.log.Check(err, "setting read deadline") - line, err := bufpool.Readline(c.br) + line, err := bufpool.Readline(c.log, c.br) if err != nil && errors.Is(err, moxio.ErrLineTooLong) { return "", fmt.Errorf("%s (%w)", err, errProtocol) } else if err != nil { @@ -629,15 +628,18 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x cmd: "(greeting)", cmdStart: time.Now(), } - c.log = xlog.MoreFields(func() []mlog.Pair { + var logmutex sync.Mutex + c.log = mlog.New("imapserver", nil).WithFunc(func() []slog.Attr { + logmutex.Lock() + defer logmutex.Unlock() now := time.Now() - l := []mlog.Pair{ - mlog.Field("cid", c.cid), - mlog.Field("delta", now.Sub(c.lastlog)), + l := []slog.Attr{ + slog.Int64("cid", c.cid), + slog.Duration("delta", now.Sub(c.lastlog)), } c.lastlog = now if c.username != "" { - l = append(l, mlog.Field("username", c.username)) + l = append(l, slog.String("username", c.username)) } return l }) @@ -662,7 +664,7 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x } } - c.log.Info("new connection", mlog.Field("remote", c.conn.RemoteAddr()), mlog.Field("local", c.conn.LocalAddr()), mlog.Field("tls", xtls), mlog.Field("listener", listenerName)) + c.log.Info("new connection", slog.Any("remote", c.conn.RemoteAddr()), slog.Any("local", c.conn.LocalAddr()), slog.Bool("tls", xtls), slog.String("listener", listenerName)) defer func() { c.conn.Close() @@ -681,7 +683,7 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x } else if err, ok := x.(error); ok && isClosed(err) { c.log.Infox("connection closed", err) } else { - c.log.Error("unhandled panic", mlog.Field("err", x)) + c.log.Error("unhandled panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Imapserver) } @@ -703,13 +705,13 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x // If remote IP/network resulted in too many authentication failures, refuse to serve. if !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) { metrics.AuthenticationRatelimitedInc("imap") - c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP)) + c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP)) c.writelinef("* BYE too many auth failures") return } if !limiterConnections.Add(c.remoteIP, time.Now(), 1) { - c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP)) + c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP)) c.writelinef("* BYE too many open connections from your ip or network") return } @@ -744,9 +746,9 @@ func (c *conn) command() { metricIMAPCommands.WithLabelValues(c.cmdMetric, result).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) }() - logFields := []mlog.Pair{ - mlog.Field("cmd", c.cmd), - mlog.Field("duration", time.Since(c.cmdStart)), + logFields := []slog.Attr{ + slog.String("cmd", c.cmd), + slog.Duration("duration", time.Since(c.cmdStart)), } c.cmd = "" @@ -761,7 +763,7 @@ func (c *conn) command() { } err, ok := x.(error) if !ok { - c.log.Error("imap command panic", append([]mlog.Pair{mlog.Field("panic", x)}, logFields...)...) + c.log.Error("imap command panic", append([]slog.Attr{slog.Any("panic", x)}, logFields...)...) result = "panic" panic(x) } @@ -786,7 +788,7 @@ func (c *conn) command() { panic(errIO) } c.log.Debugx("imap command syntax error", sxerr.err, logFields...) - c.log.Info("imap syntax error", mlog.Field("lastline", c.lastLine)) + c.log.Info("imap syntax error", slog.String("lastline", c.lastLine)) fatal := strings.HasSuffix(c.lastLine, "+}") if fatal { err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) @@ -865,7 +867,7 @@ func (c *conn) broadcast(changes []store.Change) { if len(changes) == 0 { return } - c.log.Debug("broadcast changes", mlog.Field("changes", changes)) + c.log.Debug("broadcast changes", slog.Any("changes", changes)) c.comm.Broadcast(changes) } @@ -1184,7 +1186,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { err := c.conn.SetWriteDeadline(time.Now().Add(5 * time.Minute)) c.log.Check(err, "setting write deadline") - c.log.Debug("applying changes", mlog.Field("changes", changes)) + c.log.Debug("applying changes", slog.Any("changes", changes)) // Only keep changes for the selected mailbox, and changes that are always relevant. var n []store.Change @@ -1404,7 +1406,7 @@ func (c *conn) cmdID(tag, cmd string, p *parser) { p.xempty() // We just log the client id. - c.log.Info("client id", mlog.Field("params", params)) + c.log.Info("client id", slog.Any("params", params)) // Response syntax: ../rfc/2971:243 // We send our name and version. ../rfc/2971:193 @@ -1448,7 +1450,7 @@ func (c *conn) cmdStarttls(tag, cmd string, p *parser) { } cancel() tlsversion, ciphersuite := mox.TLSInfo(tlsConn) - c.log.Debug("tls server handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite)) + c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite)) c.conn = tlsConn c.tr = moxio.NewTraceReader(c.log, "C: ", c.conn) @@ -1560,11 +1562,11 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { xusercodeErrorf("AUTHORIZATIONFAILED", "cannot assume role") } - acc, err := store.OpenEmailAuth(authc, password) + acc, err := store.OpenEmailAuth(c.log, authc, password) if err != nil { if errors.Is(err, store.ErrUnknownCredentials) { authResult = "badcreds" - c.log.Info("authentication failed", mlog.Field("username", authc)) + c.log.Info("authentication failed", slog.String("username", authc)) xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") } xusercodeErrorf("", "error") @@ -1588,11 +1590,11 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { xsyntaxErrorf("malformed cram-md5 response") } addr := t[0] - c.log.Debug("cram-md5 auth", mlog.Field("address", addr)) - acc, _, err := store.OpenEmail(addr) + c.log.Debug("cram-md5 auth", slog.String("address", addr)) + acc, _, err := store.OpenEmail(c.log, addr) if err != nil { if errors.Is(err, store.ErrUnknownCredentials) { - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") } xserverErrorf("looking up address: %v", err) @@ -1608,7 +1610,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error { password, err := bstore.QueryTx[store.Password](tx).Get() if err == bstore.ErrAbsent { - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") } if err != nil { @@ -1622,8 +1624,8 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { xcheckf(err, "tx read") }) if ipadhash == nil || opadhash == nil { - c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", mlog.Field("username", addr)) - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", addr)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") } @@ -1632,7 +1634,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { opadhash.Write(ipadhash.Sum(nil)) digest := fmt.Sprintf("%x", opadhash.Sum(nil)) if digest != t[1] { - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") } @@ -1659,8 +1661,8 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { if err != nil { xsyntaxErrorf("starting scram: %s", err) } - c.log.Debug("scram auth", mlog.Field("authentication", ss.Authentication)) - acc, _, err := store.OpenEmail(ss.Authentication) + c.log.Debug("scram auth", slog.String("authentication", ss.Authentication)) + acc, _, err := store.OpenEmail(c.log, ss.Authentication) if err != nil { // todo: we could continue scram with a generated salt, deterministically generated // from the username. that way we don't have to store anything but attackers cannot @@ -1686,7 +1688,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { xscram = password.SCRAMSHA256 } if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) { - c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", mlog.Field("address", ss.Authentication)) + c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("address", ss.Authentication)) xuserErrorf("scram not possible") } xcheckf(err, "fetching credentials") @@ -1706,7 +1708,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) { c.readline(false) // Should be "*" for cancellation. if errors.Is(err, scram.ErrInvalidProof) { authResult = "badcreds" - c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP)) xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials") } xuserErrorf("server final: %w", err) @@ -1770,13 +1772,13 @@ func (c *conn) cmdLogin(tag, cmd string, p *parser) { } }() - acc, err := store.OpenEmailAuth(userid, password) + acc, err := store.OpenEmailAuth(c.log, userid, password) if err != nil { authResult = "badcreds" var code string if errors.Is(err, store.ErrUnknownCredentials) { code = "AUTHENTICATIONFAILED" - c.log.Info("failed authentication attempt", mlog.Field("username", userid), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", userid), slog.Any("remote", c.remoteIP)) } xusercodeErrorf(code, "login failed") } @@ -2251,7 +2253,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) { for _, mID := range removeMessageIDs { p := c.account.MessagePath(mID) err := os.Remove(p) - c.log.Check(err, "removing message file for mailbox delete", mlog.Field("path", p)) + c.log.Check(err, "removing message file for mailbox delete", slog.String("path", p)) } c.ok(tag, cmd) @@ -2674,7 +2676,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { } // Read the message into a temporary file. - msgFile, err := store.CreateMessageTemp("imap-append") + msgFile, err := store.CreateMessageTemp(c.log, "imap-append") xcheckf(err, "creating temp file for message") defer func() { p := msgFile.Name() @@ -3273,7 +3275,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { } for dir := range syncDirs { - err := moxio.SyncDir(dir) + err := moxio.SyncDir(c.log, dir) xcheckf(err, "sync directory") } diff --git a/imapserver/server_test.go b/imapserver/server_test.go index db25670..86a1fb7 100644 --- a/imapserver/server_test.go +++ b/imapserver/server_test.go @@ -17,12 +17,14 @@ import ( "time" "github.com/mjl-/mox/imapclient" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/store" ) var ctxbg = context.Background() +var pkglog = mlog.New("imapserver", nil) func init() { sanityChecks = true @@ -341,10 +343,10 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn mox.Context = ctxbg mox.ConfigStaticPath = filepath.FromSlash("../testdata/imap/mox.conf") mox.MustLoadConfig(true, false) - acc, err := store.OpenAccount("mjl") + acc, err := store.OpenAccount(pkglog, "mjl") tcheck(t, err, "open account") if first { - err = acc.SetPassword("testtest") + err = acc.SetPassword(pkglog, "testtest") tcheck(t, err, "set password") } switchStop := func() {} diff --git a/import.go b/import.go index 79c855d..2beb9c1 100644 --- a/import.go +++ b/import.go @@ -16,11 +16,11 @@ import ( "time" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/mjl-/mox/config" "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) @@ -119,10 +119,9 @@ func xcmdXImport(mbox bool, c *cmd) { } defer store.Switchboard()() - xlog := mlog.New("import") cconn, sconn := net.Pipe() - clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: xlog} - serverctl := ctl{conn: sconn, r: bufio.NewReader(sconn), log: xlog} + clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: c.log} + serverctl := ctl{conn: sconn, r: bufio.NewReader(sconn), log: c.log} go servectlcmd(context.Background(), &serverctl, func() {}) ctlcmdImport(&clientctl, mbox, account, args[1], args[2]) @@ -177,7 +176,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { if mbox { kind = "mbox" } - ctl.log.Info("importing messages", mlog.Field("kind", kind), mlog.Field("account", account), mlog.Field("mailbox", mailbox), mlog.Field("source", src)) + ctl.log.Info("importing messages", slog.String("kind", kind), slog.String("account", account), slog.String("mailbox", mailbox), slog.String("source", src)) var err error var mboxf *os.File @@ -186,7 +185,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { // Open account, creating a database file if it doesn't exist yet. It must be known // in the configuration file. - a, err := store.OpenAccount(account) + a, err := store.OpenAccount(ctl.log, account) ctl.xcheck(err, "opening account") defer func() { if a != nil { @@ -222,13 +221,13 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { if mbox { mboxf, err = os.Open(src) ctl.xcheck(err, "open mbox file") - msgreader = store.NewMboxReader(store.CreateMessageTemp, src, mboxf, ctl.log) + msgreader = store.NewMboxReader(ctl.log, store.CreateMessageTemp, src, mboxf) } else { mdnewf, err = os.Open(filepath.Join(src, "new")) ctl.xcheck(err, "open subdir new of maildir") mdcurf, err = os.Open(filepath.Join(src, "cur")) ctl.xcheck(err, "open subdir cur of maildir") - msgreader = store.NewMaildirReader(store.CreateMessageTemp, mdnewf, mdcurf, ctl.log) + msgreader = store.NewMaildirReader(ctl.log, store.CreateMessageTemp, mdnewf, mdcurf) } tx, err := a.DB.Begin(ctx, true) @@ -253,7 +252,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { } if x != ctl.x { - ctl.log.Error("import error", mlog.Field("panic", fmt.Errorf("%v", x))) + ctl.log.Error("import error", slog.String("panic", fmt.Sprintf("%v", x))) debug.PrintStack() metrics.PanicInc(metrics.Import) } else { @@ -263,7 +262,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { for _, id := range deliveredIDs { p := a.MessagePath(id) err := os.Remove(p) - ctl.log.Check(err, "closing message file after import error", mlog.Field("path", p)) + ctl.log.Check(err, "closing message file after import error", slog.String("path", p)) } ctl.xerror(fmt.Sprintf("import error: %v", x)) @@ -282,7 +281,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { err := a.DeliverMessage(ctl.log, tx, m, mf, sync, notrain, nothreads) ctl.xcheck(err, "delivering message") deliveredIDs = append(deliveredIDs, m.ID) - ctl.log.Debug("delivered message", mlog.Field("id", m.ID)) + ctl.log.Debug("delivered message", slog.Int64("id", m.ID)) changes = append(changes, m.ChangeAddUID()) } @@ -319,9 +318,9 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { mb.Add(m.MailboxCounts()) // Parse message and store parsed information for later fast retrieval. - p, err := message.EnsurePart(ctl.log, false, msgf, m.Size) + p, err := message.EnsurePart(ctl.log.Logger, false, msgf, m.Size) if err != nil { - ctl.log.Infox("parsing message, continuing", err, mlog.Field("path", origPath)) + ctl.log.Infox("parsing message, continuing", err, slog.String("path", origPath)) } m.ParsedBuf, err = json.Marshal(p) ctl.xcheck(err, "marshal parsed message structure") @@ -345,7 +344,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { m.JunkFlagsForMailbox(mb, conf) if jf != nil && m.NeedsTraining() { if words, err := jf.ParseMessage(p); err != nil { - ctl.log.Infox("parsing message for updating junk filter", err, mlog.Field("parse", ""), mlog.Field("path", origPath)) + ctl.log.Infox("parsing message for updating junk filter", err, slog.String("parse", ""), slog.String("path", origPath)) } else { err = jf.Train(ctx, !m.Junk, words) ctl.xcheck(err, "training junk filter") @@ -407,7 +406,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { err = tx.Commit() ctl.xcheck(err, "commit") tx = nil - ctl.log.Info("delivered messages through import", mlog.Field("count", len(deliveredIDs))) + ctl.log.Info("delivered messages through import", slog.Int("count", len(deliveredIDs))) deliveredIDs = nil store.BroadcastChanges(a, changes) diff --git a/integration_test.go b/integration_test.go index c50c7c5..a98297c 100644 --- a/integration_test.go +++ b/integration_test.go @@ -14,6 +14,8 @@ import ( "testing" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/imapclient" "github.com/mjl-/mox/mlog" @@ -30,7 +32,7 @@ func tcheck(t *testing.T, err error, errmsg string) { } func TestDeliver(t *testing.T) { - xlog := mlog.New("integration") + log := mlog.New("integration", nil) mlog.Logfmt = true hostname, err := os.Hostname() @@ -129,7 +131,7 @@ This is the message. `, mailfrom, rcptto) msg = strings.ReplaceAll(msg, "\n", "\r\n") auth := []sasl.Client{sasl.NewClientPlain(mailfrom, password)} - c, err := smtpclient.New(mox.Context, xlog, conn, smtpclient.TLSSkip, false, ourHostname, dns.Domain{ASCII: desthost}, smtpclient.Opts{Auth: auth}) + c, err := smtpclient.New(mox.Context, log.Logger, conn, smtpclient.TLSSkip, false, ourHostname, dns.Domain{ASCII: desthost}, smtpclient.Opts{Auth: auth}) tcheck(t, err, "smtp hello") err = c.Deliver(mox.Context, mailfrom, rcptto, int64(len(msg)), strings.NewReader(msg), false, false, false) tcheck(t, err, "deliver with smtp") @@ -142,35 +144,35 @@ This is the message. tcheck(t, err, "dial submission") defer conn.Close() - xlog.Print("submitting email to moxacmepebble, waiting for imap notification at moxmail2") + log.Print("submitting email to moxacmepebble, waiting for imap notification at moxmail2") t0 := time.Now() deliver(true, true, "moxmail2.mox2.example:993", "moxtest2@mox2.example", "accountpass4321", func() { submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "moxtest2@mox2.example") }) - xlog.Print("success", mlog.Field("duration", time.Since(t0))) + log.Print("success", slog.Duration("duration", time.Since(t0))) - xlog.Print("submitting email to moxmail2, waiting for imap notification at moxacmepebble") + log.Print("submitting email to moxmail2, waiting for imap notification at moxacmepebble") t0 = time.Now() deliver(true, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() { submit(true, "moxtest2@mox2.example", "accountpass4321", "moxmail2.mox2.example:465", "moxtest1@mox1.example") }) - xlog.Print("success", mlog.Field("duration", time.Since(t0))) + log.Print("success", slog.Duration("duration", time.Since(t0))) - xlog.Print("submitting email to postfix, waiting for imap notification at moxacmepebble") + log.Print("submitting email to postfix, waiting for imap notification at moxacmepebble") t0 = time.Now() deliver(false, true, "moxacmepebble.mox1.example:993", "moxtest1@mox1.example", "accountpass1234", func() { submit(true, "moxtest1@mox1.example", "accountpass1234", "moxacmepebble.mox1.example:465", "root@postfix.example") }) - xlog.Print("success", mlog.Field("duration", time.Since(t0))) + log.Print("success", slog.Duration("duration", time.Since(t0))) - xlog.Print("submitting email to localserve") + log.Print("submitting email to localserve") t0 = time.Now() deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() { submit(false, "mox@localhost", "moxmoxmox", "localserve.mox1.example:1587", "moxtest1@mox1.example") }) - xlog.Print("success", mlog.Field("duration", time.Since(t0))) + log.Print("success", slog.Duration("duration", time.Since(t0))) - xlog.Print("submitting email to localserve") + log.Print("submitting email to localserve") t0 = time.Now() deliver(false, false, "localserve.mox1.example:1143", "mox@localhost", "moxmoxmox", func() { cmd := exec.Command("go", "run", ".", "sendmail", "mox@localhost") @@ -182,8 +184,8 @@ a message. var out strings.Builder cmd.Stdout = &out err := cmd.Run() - xlog.Print("sendmail", mlog.Field("output", out.String())) + log.Print("sendmail", slog.String("output", out.String())) tcheck(t, err, "sendmail") }) - xlog.Print("success", mlog.Field("duration", time.Since(t0))) + log.Print("success", slog.Any("duration", time.Since(t0))) } diff --git a/iprev/iprev.go b/iprev/iprev.go index 0f802be..22be3bf 100644 --- a/iprev/iprev.go +++ b/iprev/iprev.go @@ -9,6 +9,8 @@ import ( "net" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -16,7 +18,7 @@ import ( "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("iprev") +var xlog = mlog.New("iprev", nil) var ( metricIPRev = promauto.NewHistogramVec( @@ -61,7 +63,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, ip net.IP) (rstatus Stat start := time.Now() defer func() { metricIPRev.WithLabelValues(string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("iprev lookup result", rerr, mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("duration", time.Since(start))) + log.Debugx("iprev lookup result", rerr, slog.Any("ip", ip), slog.Any("status", rstatus), slog.Duration("duration", time.Since(start))) }() revNames, result, revErr := dns.WithPackage(resolver, "iprev").LookupAddr(ctx, ip.String()) diff --git a/junk.go b/junk.go index 5871bff..0aabc3d 100644 --- a/junk.go +++ b/junk.go @@ -93,7 +93,7 @@ func cmdJunkTrain(c *cmd) { } a.SetLogLevel() - f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath)) + f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath)) defer func() { if err := f.Close(); err != nil { log.Printf("closing junk filter: %v", err) @@ -122,7 +122,7 @@ func cmdJunkCheck(c *cmd) { } a.SetLogLevel() - f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false)) + f := must(junk.OpenFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath, false)) defer func() { if err := f.Close(); err != nil { log.Printf("closing junk filter: %v", err) @@ -146,7 +146,7 @@ func cmdJunkTest(c *cmd) { } a.SetLogLevel() - f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false)) + f := must(junk.OpenFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath, false)) defer func() { if err := f.Close(); err != nil { log.Printf("closing junk filter: %v", err) @@ -202,7 +202,7 @@ messages are shuffled, with optional random seed.` } a.SetLogLevel() - f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath)) + f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath)) defer func() { if err := f.Close(); err != nil { log.Printf("closing junk filter: %v", err) @@ -293,7 +293,7 @@ func cmdJunkPlay(c *cmd) { } a.SetLogLevel() - f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath)) + f := must(junk.NewFilter(context.Background(), c.log, a.params, a.databasePath, a.bloomfilterPath)) defer func() { if err := f.Close(); err != nil { log.Printf("closing junk filter: %v", err) @@ -310,8 +310,6 @@ func cmdJunkPlay(c *cmd) { var nbad, nnodate, nham, nspam, nsent int - jlog := mlog.New("junkplay") - scanDir := func(dir string, ham, sent bool) { for _, name := range listDir(dir) { path := filepath.Join(dir, name) @@ -319,7 +317,7 @@ func cmdJunkPlay(c *cmd) { xcheckf(err, "open %q", path) fi, err := mf.Stat() xcheckf(err, "stat %q", path) - p, err := message.EnsurePart(jlog, false, mf, fi.Size()) + p, err := message.EnsurePart(c.log.Logger, false, mf, fi.Size()) if err != nil { nbad++ if err := mf.Close(); err != nil { @@ -399,7 +397,7 @@ func cmdJunkPlay(c *cmd) { }() fi, err := mf.Stat() xcheckf(err, "stat %q", path) - p, err := message.EnsurePart(jlog, false, mf, fi.Size()) + p, err := message.EnsurePart(c.log.Logger, false, mf, fi.Size()) if err != nil { log.Printf("bad sent message %q: %s", path, err) return diff --git a/junk/filter.go b/junk/filter.go index 574ec59..471e576 100644 --- a/junk/filter.go +++ b/junk/filter.go @@ -21,6 +21,8 @@ import ( "sort" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/message" @@ -28,8 +30,6 @@ import ( ) var ( - xlog = mlog.New("junk") - // errBadContentType = errors.New("bad content-type") // sure sign of spam, todo: use this error errClosed = errors.New("filter is closed") ) @@ -62,7 +62,7 @@ var DBTypes = []any{wordscore{}} // Stored in DB. type Filter struct { Params - log *mlog.Log // For logging cid. + log mlog.Log // For logging cid. closed bool modified bool // Whether any modifications are pending. Cleared by Save. hams, spams uint32 // Message count, stored in db under word "-". @@ -112,7 +112,7 @@ func (f *Filter) Close() error { return err } -func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) { +func OpenFilter(ctx context.Context, log mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) { var bloom *Bloom if loadBloom { var err error @@ -160,7 +160,7 @@ func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloom // filter is marked as new until the first save, will be done automatically if // TrainDirs is called. If the bloom and/or database files exist, an error is // returned. -func NewFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) { +func NewFilter(ctx context.Context, log mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) { var err error if _, err := os.Stat(bloomPath); err == nil { return nil, fmt.Errorf("bloom filter already exists on disk: %s", bloomPath) @@ -220,7 +220,7 @@ func openBloom(path string) (*Bloom, error) { return NewBloom(buf, bloomK) } -func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr error) { +func newDB(ctx context.Context, log mlog.Log, path string) (db *bstore.DB, rerr error) { // Remove any existing files. os.Remove(path) @@ -272,7 +272,7 @@ func (f *Filter) Save() error { return words[i] < words[j] }) - f.log.Debug("inserting words in junkfilter db", mlog.Field("words", len(f.changed))) + f.log.Debug("inserting words in junkfilter db", slog.Any("words", len(f.changed))) // start := time.Now() if f.isNew { if err := f.db.HintAppend(true, wordscore{}); err != nil { @@ -318,7 +318,7 @@ func (f *Filter) Save() error { f.changed = map[string]word{} f.modified = false f.isNew = false - // f.log.Info("wrote filter to db", mlog.Field("duration", time.Since(start))) + // f.log.Info("wrote filter to db", slog.Any("duration", time.Since(start))) return nil } @@ -378,7 +378,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) ( expect[w] = struct{}{} } if len(unknowns) > 0 { - f.log.Debug("unknown words in bloom filter, showing max 50", mlog.Field("words", unknowns), mlog.Field("totalunknown", totalUnknown), mlog.Field("totalwords", len(words))) + f.log.Debug("unknown words in bloom filter, showing max 50", slog.Any("words", unknowns), slog.Any("totalunknown", totalUnknown), slog.Any("totalwords", len(words))) } // Fetch words from database. @@ -391,7 +391,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) ( delete(expect, w) f.cache[w] = c } - f.log.Debug("unknown words in db", mlog.Field("words", expect), mlog.Field("totalunknown", len(expect)), mlog.Field("totalwords", len(words))) + f.log.Debug("unknown words in db", slog.Any("words", expect), slog.Any("totalunknown", len(expect)), slog.Any("totalwords", len(words))) } for w := range words { @@ -474,7 +474,7 @@ func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) ( eta += math.Log(1-x.R) - math.Log(x.R) } - f.log.Debug("top words", mlog.Field("hams", topHam), mlog.Field("spams", topSpam)) + f.log.Debug("top words", slog.Any("hams", topHam), slog.Any("spams", topSpam)) prob := 1 / (1 + math.Pow(math.E, eta)) return prob, len(topHam), len(topSpam), nil @@ -502,7 +502,7 @@ func (f *Filter) ClassifyMessagePath(ctx context.Context, path string) (probabil } func (f *Filter) ClassifyMessageReader(ctx context.Context, mf io.ReaderAt, size int64) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) { - m, err := message.EnsurePart(f.log, false, mf, size) + m, err := message.EnsurePart(f.log.Logger, false, mf, size) if err != nil && errors.Is(err, message.ErrBadContentType) { // Invalid content-type header is a sure sign of spam. //f.log.Infox("parsing content", err) @@ -568,7 +568,7 @@ func (f *Filter) Train(ctx context.Context, ham bool, words map[string]struct{}) } func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error { - p, _ := message.EnsurePart(f.log, false, r, size) + p, _ := message.EnsurePart(f.log.Logger, false, r, size) words, err := f.ParseMessage(p) if err != nil { return fmt.Errorf("parsing mail contents: %v", err) @@ -577,7 +577,7 @@ func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ha } func (f *Filter) UntrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error { - p, _ := message.EnsurePart(f.log, false, r, size) + p, _ := message.EnsurePart(f.log.Logger, false, r, size) words, err := f.ParseMessage(p) if err != nil { return fmt.Errorf("parsing mail contents: %v", err) @@ -648,7 +648,7 @@ func (f *Filter) TrainDir(dir string, files []string, ham bool) (n, malformed ui p := filepath.Join(dir, name) valid, words, err := f.tokenizeMail(p) if err != nil { - // f.log.Infox("tokenizing mail", err, mlog.Field("path", p)) + // f.log.Infox("tokenizing mail", err, slog.Any("path", p)) malformed++ continue } @@ -720,21 +720,20 @@ func (f *Filter) TrainDirs(hamDir, sentDir, spamDir string, hamFiles, sentFiles, dbSize := f.fileSize(f.dbPath) bloomSize := f.fileSize(f.bloomPath) - fields := []mlog.Pair{ - mlog.Field("hams", hams), - mlog.Field("hamtime", tham), - mlog.Field("hammalformed", hamMalformed), - mlog.Field("sent", sent), - mlog.Field("senttime", tsent), - mlog.Field("sentmalformed", sentMalformed), - mlog.Field("spams", f.spams), - mlog.Field("spamtime", tspam), - mlog.Field("spammalformed", spamMalformed), - mlog.Field("dbsize", fmt.Sprintf("%.1fmb", float64(dbSize)/(1024*1024))), - mlog.Field("bloomsize", fmt.Sprintf("%.1fmb", float64(bloomSize)/(1024*1024))), - mlog.Field("bloom1ratio", fmt.Sprintf("%.4f", float64(f.bloom.Ones())/float64(len(f.bloom.Bytes())*8))), - } - xlog.Print("training done", fields...) + f.log.Print("training done", + slog.Any("hams", hams), + slog.Any("hamtime", tham), + slog.Any("hammalformed", hamMalformed), + slog.Any("sent", sent), + slog.Any("senttime", tsent), + slog.Any("sentmalformed", sentMalformed), + slog.Any("spams", f.spams), + slog.Any("spamtime", tspam), + slog.Any("spammalformed", spamMalformed), + slog.Any("dbsize", fmt.Sprintf("%.1fmb", float64(dbSize)/(1024*1024))), + slog.Any("bloomsize", fmt.Sprintf("%.1fmb", float64(bloomSize)/(1024*1024))), + slog.Any("bloom1ratio", fmt.Sprintf("%.4f", float64(f.bloom.Ones())/float64(len(f.bloom.Bytes())*8))), + ) return nil } @@ -742,7 +741,7 @@ func (f *Filter) TrainDirs(hamDir, sentDir, spamDir string, hamFiles, sentFiles, func (f *Filter) fileSize(p string) int { fi, err := os.Stat(p) if err != nil { - f.log.Infox("stat", err, mlog.Field("path", p)) + f.log.Infox("stat", err, slog.Any("path", p)) return 0 } return int(fi.Size()) diff --git a/junk/filter_test.go b/junk/filter_test.go index f0ea03b..bb67c0f 100644 --- a/junk/filter_test.go +++ b/junk/filter_test.go @@ -32,7 +32,7 @@ func tlistdir(t *testing.T, name string) []string { } func TestFilter(t *testing.T) { - log := mlog.New("junk") + log := mlog.New("junk", nil) params := Params{ Onegrams: true, Twograms: true, diff --git a/junk/parse.go b/junk/parse.go index d5b8859..3336198 100644 --- a/junk/parse.go +++ b/junk/parse.go @@ -31,7 +31,7 @@ func (f *Filter) tokenizeMail(path string) (bool, map[string]struct{}, error) { if err != nil { return false, nil, err } - p, _ := message.EnsurePart(f.log, false, mf, fi.Size()) + p, _ := message.EnsurePart(f.log.Logger, false, mf, fi.Size()) words, err := f.ParseMessage(p) return true, words, err } diff --git a/junk/parse_test.go b/junk/parse_test.go index 87ca9b2..ed87a7d 100644 --- a/junk/parse_test.go +++ b/junk/parse_test.go @@ -4,6 +4,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/mjl-/mox/mlog" ) func FuzzParseMessage(f *testing.F) { @@ -24,7 +26,8 @@ func FuzzParseMessage(f *testing.F) { os.Remove(dbPath) os.Remove(bloomPath) params := Params{Twograms: true} - jf, err := NewFilter(ctxbg, xlog, params, dbPath, bloomPath) + log := mlog.New("junk", nil) + jf, err := NewFilter(ctxbg, log, params, dbPath, bloomPath) if err != nil { f.Fatalf("new filter: %v", err) } diff --git a/localserve.go b/localserve.go index 5768600..cfcc082 100644 --- a/localserve.go +++ b/localserve.go @@ -20,6 +20,7 @@ import ( "time" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slog" "github.com/mjl-/sconf" @@ -75,18 +76,17 @@ during those commands instead of during "data". c.Usage() } - log := mlog.New("localserve") - + log := c.log mox.FilesImmediate = true if initOnly { if _, err := os.Stat(dir); err == nil { log.Print("warning: directory for configuration files already exists, continuing") } - log.Print("creating mox localserve config", mlog.Field("dir", dir)) + log.Print("creating mox localserve config", slog.String("dir", dir)) err := writeLocalConfig(log, dir, ip) if err != nil { - log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir)) + log.Fatalx("creating mox localserve config", err, slog.String("dir", dir)) } return } @@ -96,12 +96,12 @@ during those commands instead of during "data". if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { err := writeLocalConfig(log, dir, ip) if err != nil { - log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir)) + log.Fatalx("creating mox localserve config", err, slog.String("dir", dir)) } } else if err != nil { - log.Fatalx("stat config dir", err, mlog.Field("dir", dir)) + log.Fatalx("stat config dir", err, slog.String("dir", dir)) } else if err := localLoadConfig(log, dir); err != nil { - log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, mlog.Field("dir", dir)) + log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, slog.String("dir", dir)) } else if ip != "" { log.Fatal("can only use -ip when writing a new config file") } else { @@ -112,7 +112,7 @@ during those commands instead of during "data". mox.Conf.Log[""] = level mlog.SetConfig(mox.Conf.Log) } else if loglevel != "" && !ok { - log.Fatal("unknown loglevel", mlog.Field("loglevel", loglevel)) + log.Fatal("unknown loglevel", slog.String("loglevel", loglevel)) } // Initialize receivedid. @@ -201,7 +201,7 @@ during those commands instead of during "data". sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) sig := <-sigc - log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig)) + log.Print("shutting down, waiting max 3s for existing connections", slog.Any("signal", sig)) shutdown(log) if num, ok := sig.(syscall.Signal); ok { os.Exit(int(num)) @@ -210,7 +210,7 @@ during those commands instead of during "data". } } -func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) { +func writeLocalConfig(log mlog.Log, dir, ip string) (rerr error) { defer func() { x := recover() if x != nil { @@ -220,7 +220,7 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) { } if rerr != nil { err := os.RemoveAll(dir) - log.Check(err, "removing config directory", mlog.Field("dir", dir)) + log.Check(err, "removing config directory", slog.String("dir", dir)) } }() @@ -430,10 +430,10 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) { xcheck(err, "loading config") // Set password on account. - a, _, err := store.OpenEmail("mox@localhost") + a, _, err := store.OpenEmail(log, "mox@localhost") xcheck(err, "opening account to set password") password := "moxmoxmox" - err = a.SetPassword(password) + err = a.SetPassword(log, password) xcheck(err, "setting password") err = a.Close() xcheck(err, "closing account") @@ -442,10 +442,10 @@ func writeLocalConfig(log *mlog.Log, dir, ip string) (rerr error) { return nil } -func localLoadConfig(log *mlog.Log, dir string) error { +func localLoadConfig(log mlog.Log, dir string) error { mox.ConfigStaticPath = filepath.Join(dir, "mox.conf") mox.ConfigDynamicPath = filepath.Join(dir, "domains.conf") - errs := mox.LoadConfig(context.Background(), true, false) + errs := mox.LoadConfig(context.Background(), log, true, false) if len(errs) > 1 { log.Error("loading config generated config file: multiple errors") for _, err := range errs { diff --git a/main.go b/main.go index 16489f5..6739508 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,7 @@ import ( "golang.org/x/crypto/bcrypt" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/mjl-/adns" @@ -211,6 +212,8 @@ type cmd struct { params string // Arguments to command. Multiple lines possible. help string // Additional explanation. First line is synopsis, the rest is only printed for an explicit help/usage for that command. args []string + + log mlog.Log } func (c *cmd) Parse() []string { @@ -388,7 +391,7 @@ func mustLoadConfig() { mox.Conf.Log[""] = level mlog.SetConfig(mox.Conf.Log) } else if loglevel != "" && !ok { - log.Fatal("unknown loglevel", mlog.Field("loglevel", loglevel)) + log.Fatal("unknown loglevel", slog.String("loglevel", loglevel)) } if pedantic { moxvar.Pedantic = true @@ -413,6 +416,7 @@ func main() { c := &cmd{ flag: flag.NewFlagSet("sendmail", flag.ExitOnError), flagArgs: os.Args[1:], + log: mlog.New("sendmail", nil), } cmdSendmail(c) return @@ -464,6 +468,7 @@ next: } c.flag = flag.NewFlagSet("mox "+strings.Join(c.words, " "), flag.ExitOnError) c.flagArgs = args[len(c.words):] + c.log = mlog.New(strings.Join(c.words, ""), nil) c.fn(&c) return } @@ -538,7 +543,7 @@ are printed. mox.FilesImmediate = true - _, errs := mox.ParseConfig(context.Background(), mox.ConfigStaticPath, true, true, false) + _, errs := mox.ParseConfig(context.Background(), c.log, mox.ConfigStaticPath, true, true, false) if len(errs) > 1 { log.Printf("multiple errors:") for _, err := range errs { @@ -1596,7 +1601,7 @@ connection. } resolver := dns.StrictResolver{Pkg: "danedial"} - conn, record, err := dane.Dial(context.Background(), resolver, "tcp", args[0], allowedUsages) + conn, record, err := dane.Dial(context.Background(), c.log.Logger, resolver, "tcp", args[0], allowedUsages) xcheckf(err, "dial") log.Printf("(connected, verified with %s)", record) @@ -1644,8 +1649,6 @@ sharing most of its code. origNextHop, err := dns.ParseDomain(args[0]) xcheckf(err, "parse domain") - clog := mlog.New("danedialmx") - ctxbg := context.Background() resolver := dns.StrictResolver{} @@ -1655,7 +1658,7 @@ sharing most of its code. var hosts []dns.IPDomain if len(args) == 1 { var permanent bool - haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err = smtpclient.GatherDestinations(ctxbg, clog, resolver, dns.IPDomain{Domain: origNextHop}) + haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err = smtpclient.GatherDestinations(ctxbg, c.log.Logger, resolver, dns.IPDomain{Domain: origNextHop}) status := "temporary" if permanent { status = "permanent" @@ -1706,7 +1709,7 @@ sharing most of its code. log.Printf("attempting to connect to %s", host) - authentic, expandedAuthentic, expandedHost, ips, _, err := smtpclient.GatherIPs(ctxbg, clog, resolver, host, dialedIPs) + authentic, expandedAuthentic, expandedHost, ips, _, err := smtpclient.GatherIPs(ctxbg, c.log.Logger, resolver, host, dialedIPs) if err != nil { log.Printf("resolving ips for %s: %v, skipping", host, err) continue @@ -1724,7 +1727,7 @@ sharing most of its code. } log.Printf("host %s resolved to ips %s, looking up tlsa records", host, ips) - daneRequired, daneRecords, tlsaBaseDomain, err := smtpclient.GatherTLSA(ctxbg, clog, resolver, host.Domain, expandedAuthentic, expandedHost) + daneRequired, daneRecords, tlsaBaseDomain, err := smtpclient.GatherTLSA(ctxbg, c.log.Logger, resolver, host.Domain, expandedAuthentic, expandedHost) if err != nil { log.Printf("looking up tlsa records: %s, skipping", err) continue @@ -1753,7 +1756,7 @@ sharing most of its code. log.Printf("gathered valid tls certificate names for potential verification with dane-ta: %s", strings.Join(l, ", ")) dialer := &net.Dialer{Timeout: 5 * time.Second} - conn, _, err := smtpclient.Dial(ctxbg, clog, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs) + conn, _, err := smtpclient.Dial(ctxbg, c.log.Logger, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs) if err != nil { log.Printf("dial %s: %v, skipping", expandedHost, err) continue @@ -1768,7 +1771,7 @@ sharing most of its code. RootCAs: mox.Conf.Static.TLS.CertPool, } tlsPKIX := false - sc, err := smtpclient.New(ctxbg, clog, conn, tlsMode, tlsPKIX, ehloDomain, tlsHostnames[0], opts) + sc, err := smtpclient.New(ctxbg, c.log.Logger, conn, tlsMode, tlsPKIX, ehloDomain, tlsHostnames[0], opts) if err != nil { log.Printf("setting up smtp session: %v, skipping", err) conn.Close() @@ -2175,7 +2178,7 @@ that was passed. msgf, err := os.Open(args[0]) xcheckf(err, "open message") - results, err := dkim.Verify(context.Background(), dns.StrictResolver{}, false, dkim.DefaultPolicy, msgf, true) + results, err := dkim.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, false, dkim.DefaultPolicy, msgf, true) xcheckf(err, "dkim verify") for _, result := range results { @@ -2214,13 +2217,11 @@ headers prepended. c.Usage() } - clog := mlog.New("dkimsign") - msgf, err := os.Open(args[0]) xcheckf(err, "open message") defer msgf.Close() - p, err := message.Parse(clog, true, msgf) + p, err := message.Parse(c.log.Logger, true, msgf) xcheckf(err, "parsing message") if len(p.Envelope.From) != 1 { @@ -2237,7 +2238,7 @@ headers prepended. log.Fatalf("domain %s not configured", dom) } - headers, err := dkim.Sign(context.Background(), localpart, dom, domConf.DKIM, false, msgf) + headers, err := dkim.Sign(context.Background(), c.log.Logger, localpart, dom, domConf.DKIM, false, msgf) xcheckf(err, "signing message with dkim") if headers == "" { log.Fatalf("no DKIM configured for domain %s", dom) @@ -2259,7 +2260,7 @@ func cmdDKIMLookup(c *cmd) { selector := xparseDomain(args[0], "selector") domain := xparseDomain(args[1], "domain") - status, record, txt, authentic, err := dkim.Lookup(context.Background(), dns.StrictResolver{}, selector, domain) + status, record, txt, authentic, err := dkim.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, selector, domain) if err != nil { fmt.Printf("error: %s\n", err) } @@ -2299,7 +2300,7 @@ func cmdDMARCLookup(c *cmd) { } fromdomain := xparseDomain(args[0], "domain") - _, domain, _, txt, authentic, err := dmarc.Lookup(context.Background(), dns.StrictResolver{}, fromdomain) + _, domain, _, txt, authentic, err := dmarc.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, fromdomain) xcheckf(err, "dmarc lookup domain %s", fromdomain) fmt.Printf("dmarc record at domain %s: %s\n", domain, txt) fmt.Printf("(%s)\n", dnssecStatus(authentic)) @@ -2359,7 +2360,7 @@ can be found in message headers. if heloDomain != nil { spfArgs.HelloDomain = dns.IPDomain{Domain: *heloDomain} } - rspf, spfDomain, expl, authentic, err := spf.Verify(context.Background(), dns.StrictResolver{}, spfArgs) + rspf, spfDomain, expl, authentic, err := spf.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, spfArgs) if err != nil { log.Printf("spf verify: %v (explanation: %q, authentic %v)", err, expl, authentic) } else { @@ -2377,17 +2378,17 @@ can be found in message headers. data, err := io.ReadAll(os.Stdin) xcheckf(err, "read message") - dmarcFrom, _, _, err := message.From(mlog.New("dmarcverify"), false, bytes.NewReader(data)) + dmarcFrom, _, _, err := message.From(c.log.Logger, false, bytes.NewReader(data)) xcheckf(err, "extract dmarc from message") const ignoreTestMode = false - dkimResults, err := dkim.Verify(context.Background(), dns.StrictResolver{}, true, func(*dkim.Sig) error { return nil }, bytes.NewReader(data), ignoreTestMode) + dkimResults, err := dkim.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, true, func(*dkim.Sig) error { return nil }, bytes.NewReader(data), ignoreTestMode) xcheckf(err, "dkim verify") for _, r := range dkimResults { fmt.Printf("dkim result: %q (err %v)\n", r.Status, r.Err) } - _, result := dmarc.Verify(context.Background(), dns.StrictResolver{}, dmarcFrom.Domain, dkimResults, spfStatus, spfIdentity, false) + _, result := dmarc.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, dmarcFrom.Domain, dkimResults, spfStatus, spfIdentity, false) xcheckf(result.Err, "dmarc verify") fmt.Printf("dmarc from: %s\ndmarc status: %q\ndmarc reject: %v\ncmarc record: %s\n", dmarcFrom, result.Status, result.Reject, result.Record) } @@ -2408,7 +2409,7 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at } dom := xparseDomain(args[0], "domain") - _, domain, record, txt, authentic, err := dmarc.Lookup(context.Background(), dns.StrictResolver{}, dom) + _, domain, record, txt, authentic, err := dmarc.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, dom) xcheckf(err, "dmarc lookup domain %s", dom) fmt.Printf("dmarc record at domain %s: %q\n", domain, txt) fmt.Printf("(%s)\n", dnssecStatus(authentic)) @@ -2439,12 +2440,12 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at return } - if publicsuffix.Lookup(context.Background(), dom) == publicsuffix.Lookup(context.Background(), destdom) { + if publicsuffix.Lookup(context.Background(), c.log.Logger, dom) == publicsuffix.Lookup(context.Background(), c.log.Logger, destdom) { printResult("pass (same organizational domain)") return } - accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom) + accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), c.log.Logger, dns.StrictResolver{}, domain, destdom) var txtstr string txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII) if len(txts) == 0 { @@ -2486,12 +2487,10 @@ understand email deliverability problems. c.Usage() } - clog := mlog.New("dmarcparsereportmsg") - for _, arg := range args { f, err := os.Open(arg) xcheckf(err, "open %q", arg) - feedback, err := dmarcrpt.ParseMessageReport(clog, f) + feedback, err := dmarcrpt.ParseMessageReport(c.log.Logger, f) xcheckf(err, "parse report in %q", arg) meta := feedback.ReportMetadata fmt.Printf("Report: period %s-%s, organisation %q, reportID %q, %s\n", time.Unix(meta.DateRange.Begin, 0).UTC().String(), time.Unix(meta.DateRange.End, 0).UTC().String(), meta.OrgName, meta.ReportID, meta.Email) @@ -2540,11 +2539,9 @@ func cmdDMARCDBAddReport(c *cmd) { mustLoadConfig() - clog := mlog.New("dmarcdbaddreport") - fromdomain := xparseDomain(args[0], "domain") fmt.Fprintln(os.Stderr, "reading report message from stdin") - report, err := dmarcrpt.ParseMessageReport(clog, os.Stdin) + report, err := dmarcrpt.ParseMessageReport(c.log.Logger, os.Stdin) xcheckf(err, "parse message") err = dmarcdb.AddReport(context.Background(), report, fromdomain) xcheckf(err, "add dmarc report") @@ -2565,7 +2562,7 @@ successfully used TLS, and how what kind of errors occurred otherwise. } d := xparseDomain(args[0], "domain") - _, txt, err := tlsrpt.Lookup(context.Background(), dns.StrictResolver{}, d) + _, txt, err := tlsrpt.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, d) xcheckf(err, "tlsrpt lookup for %s", d) fmt.Println(txt) } @@ -2581,12 +2578,10 @@ The report is printed in formatted JSON. c.Usage() } - clog := mlog.New("tlsrptparsereportmsg") - for _, arg := range args { f, err := os.Open(arg) xcheckf(err, "open %q", arg) - report, err := tlsrpt.ParseMessage(clog, f) + report, err := tlsrpt.ParseMessage(c.log.Logger, f) xcheckf(err, "parse report in %q", arg) // todo future: only print the highlights? enc := json.NewEncoder(os.Stdout) @@ -2622,7 +2617,7 @@ printed. LocalIP: net.ParseIP("127.0.0.1"), LocalHostname: dns.Domain{ASCII: "localhost"}, } - r, _, explanation, authentic, err := spf.Verify(context.Background(), dns.StrictResolver{}, spfargs) + r, _, explanation, authentic, err := spf.Verify(context.Background(), c.log.Logger, dns.StrictResolver{}, spfargs) if err != nil { fmt.Printf("error: %s\n", err) } @@ -2656,7 +2651,7 @@ func cmdSPFLookup(c *cmd) { } domain := xparseDomain(args[0], "domain") - _, txt, _, authentic, err := spf.Lookup(context.Background(), dns.StrictResolver{}, domain) + _, txt, _, authentic, err := spf.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, domain) xcheckf(err, "spf lookup for %s", domain) fmt.Println(txt) fmt.Printf("(%s)\n", dnssecStatus(authentic)) @@ -2680,7 +2675,7 @@ should be used, and how long the policy can be cached. domain := xparseDomain(args[0], "domain") - record, policy, _, err := mtasts.Get(context.Background(), dns.StrictResolver{}, domain) + record, policy, _, err := mtasts.Get(context.Background(), c.log.Logger, dns.StrictResolver{}, domain) if err != nil { fmt.Printf("error: %s\n", err) } @@ -2729,13 +2724,11 @@ func cmdTLSRPTDBAddReport(c *cmd) { mustLoadConfig() - clog := mlog.New("tlsrptdbaddreport") - // First read message, to get the From-header. Then parse it as TLSRPT. fmt.Fprintln(os.Stderr, "reading report message from stdin") buf, err := io.ReadAll(os.Stdin) xcheckf(err, "reading message") - part, err := message.Parse(clog, true, bytes.NewReader(buf)) + part, err := message.Parse(c.log.Logger, true, bytes.NewReader(buf)) xcheckf(err, "parsing message") if part.Envelope == nil || len(part.Envelope.From) != 1 { log.Fatalf("message must have one From-header") @@ -2743,11 +2736,11 @@ func cmdTLSRPTDBAddReport(c *cmd) { from := part.Envelope.From[0] domain := xparseDomain(from.Host, "domain") - report, err := tlsrpt.ParseMessage(clog, bytes.NewReader(buf)) + report, err := tlsrpt.ParseMessage(c.log.Logger, bytes.NewReader(buf)) xcheckf(err, "parsing tls report in message") mailfrom := from.User + "@" + from.Host // todo future: should escape and such - err = tlsrptdb.AddReport(context.Background(), domain, mailfrom, hostReport, report) + err = tlsrptdb.AddReport(context.Background(), c.log, domain, mailfrom, hostReport, report) xcheckf(err, "add tls report to database") } @@ -2766,7 +2759,7 @@ URL with more information. zone := xparseDomain(args[0], "zone") ip := xparseIP(args[1], "ip") - status, explanation, err := dnsbl.Lookup(context.Background(), dns.StrictResolver{}, zone, ip) + status, explanation, err := dnsbl.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, zone, ip) fmt.Printf("status: %s\n", status) if status == dnsbl.StatusFail { fmt.Printf("explanation: %q\n", explanation) @@ -2789,7 +2782,7 @@ The health of a DNS blocklist can be checked by querying for 127.0.0.1 and } zone := xparseDomain(args[0], "zone") - err := dnsbl.CheckHealth(context.Background(), dns.StrictResolver{}, zone) + err := dnsbl.CheckHealth(context.Background(), c.log.Logger, dns.StrictResolver{}, zone) xcheckf(err, "unhealthy") fmt.Println("healthy") } @@ -2814,12 +2807,12 @@ printed. fmt.Printf("last known version: %s\n", lastknown) fmt.Printf("current version: %s\n", current) } - latest, _, err := updates.Lookup(context.Background(), dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain}) + latest, _, err := updates.Lookup(context.Background(), c.log.Logger, dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain}) xcheckf(err, "lookup of latest version") fmt.Printf("latest version: %s\n", latest) if latest.After(current) { - changelog, err := updates.FetchChangelog(context.Background(), changelogURL, current, changelogPubKey) + changelog, err := updates.FetchChangelog(context.Background(), c.log.Logger, changelogURL, current, changelogPubKey) xcheckf(err, "fetching changelog") if len(changelog.Changes) == 0 { log.Printf("no changes in changelog") @@ -2884,7 +2877,7 @@ open, or is not running. } mustLoadConfig() - a, err := store.OpenAccount(args[0]) + a, err := store.OpenAccount(c.log, args[0]) xcheckf(err, "open account") defer func() { if err := a.Close(); err != nil { @@ -2942,7 +2935,7 @@ open, or is not running. } mustLoadConfig() - a, err := store.OpenAccount(args[0]) + a, err := store.OpenAccount(c.log, args[0]) xcheckf(err, "open account") defer func() { if err := a.Close(); err != nil { @@ -3036,7 +3029,7 @@ open, or is not running. } mustLoadConfig() - a, err := store.OpenAccount(args[0]) + a, err := store.OpenAccount(c.log, args[0]) xcheckf(err, "open account") defer func() { if err := a.Close(); err != nil { @@ -3156,10 +3149,8 @@ func cmdEnsureParsed(c *cmd) { c.Usage() } - clog := mlog.New("ensureparsed") - mustLoadConfig() - a, err := store.OpenAccount(args[0]) + a, err := store.OpenAccount(c.log, args[0]) xcheckf(err, "open account") defer func() { if err := a.Close(); err != nil { @@ -3180,7 +3171,7 @@ func cmdEnsureParsed(c *cmd) { } for _, m := range l { mr := a.MessageReader(m) - p, err := message.EnsurePart(clog, false, mr, m.Size) + p, err := message.EnsurePart(c.log.Logger, false, mr, m.Size) if err != nil { log.Printf("parsing message %d: %v (continuing)", m.ID, err) } @@ -3233,15 +3224,13 @@ func cmdMessageParse(c *cmd) { c.Usage() } - clog := mlog.New("messageparse") - f, err := os.Open(args[0]) xcheckf(err, "open") defer f.Close() - part, err := message.Parse(clog, false, f) + part, err := message.Parse(c.log.Logger, false, f) xcheckf(err, "parsing message") - err = part.Walk(clog, nil) + err = part.Walk(c.log.Logger, nil) xcheckf(err, "parsing nested parts") enc := json.NewEncoder(os.Stdout) enc.SetIndent("", "\t") @@ -3262,15 +3251,13 @@ Opens database files directly, not going through a running mox instance. c.Usage() } - clog := mlog.New("openaccounts") - dataDir := filepath.Clean(args[0]) for _, accName := range args[1:] { accDir := filepath.Join(dataDir, "accounts", accName) log.Printf("opening account %s...", accDir) - a, err := store.OpenAccountDB(accDir, accName) + a, err := store.OpenAccountDB(c.log, accDir, accName) xcheckf(err, "open account %s", accName) - err = a.ThreadingWait(clog) + err = a.ThreadingWait(c.log) xcheckf(err, "wait for threading upgrade to complete for %s", accName) err = a.Close() xcheckf(err, "close account %s", accName) @@ -3363,7 +3350,7 @@ Opens database files directly, not going through a running mox instance. for _, accName := range args[1:] { accDir := filepath.Join(dataDir, "accounts", accName) log.Printf("opening account %s...", accDir) - a, err := store.OpenAccountDB(accDir, accName) + a, err := store.OpenAccountDB(c.log, accDir, accName) xcheckf(err, "open account %s", accName) prepareMessages := func(in, out chan moxio.Work[store.Message, threadPrep]) { diff --git a/message/from.go b/message/from.go index dc129c8..0d2a1b3 100644 --- a/message/from.go +++ b/message/from.go @@ -5,6 +5,8 @@ import ( "io" "net/textproto" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/smtp" @@ -17,12 +19,14 @@ import ( // From headers may be present. From returns an error if there is not exactly // one address. This address can be used for evaluating a DMARC policy against // SPF and DKIM results. -func From(log *mlog.Log, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) { +func From(elog *slog.Logger, strict bool, r io.ReaderAt) (raddr smtp.Address, envelope *Envelope, header textproto.MIMEHeader, rerr error) { + log := mlog.New("message", elog) + // ../rfc/7489:1243 // todo: only allow utf8 if enabled in session/message? - p, err := Parse(log, strict, r) + p, err := Parse(log.Logger, strict, r) if err != nil { // todo: should we continue with p, perhaps headers can be parsed? return raddr, nil, nil, fmt.Errorf("parsing message: %v", err) diff --git a/message/part.go b/message/part.go index fd2e9e8..4cb6ae8 100644 --- a/message/part.go +++ b/message/part.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "golang.org/x/exp/slog" "golang.org/x/text/encoding/ianaindex" "github.com/mjl-/mox/mlog" @@ -111,7 +112,8 @@ type Address struct { // // If strict is set, fewer attempts are made to continue parsing when errors are // encountered, such as with invalid content-type headers or bare carriage returns. -func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) { +func Parse(elog *slog.Logger, strict bool, r io.ReaderAt) (Part, error) { + log := mlog.New("message", elog) return newPart(log, strict, r, 0, nil) } @@ -122,10 +124,11 @@ func Parse(log *mlog.Log, strict bool, r io.ReaderAt) (Part, error) { // // If strict is set, fewer attempts are made to continue parsing when errors are // encountered, such as with invalid content-type headers or bare carriage returns. -func EnsurePart(log *mlog.Log, strict bool, r io.ReaderAt, size int64) (Part, error) { - p, err := Parse(log, strict, r) +func EnsurePart(elog *slog.Logger, strict bool, r io.ReaderAt, size int64) (Part, error) { + log := mlog.New("message", elog) + p, err := Parse(log.Logger, strict, r) if err == nil { - err = p.Walk(log, nil) + err = p.Walk(log.Logger, nil) } if err != nil { np, err2 := fallbackPart(p, r, size) @@ -185,7 +188,9 @@ func (p *Part) SetMessageReaderAt() error { } // Walk through message, decoding along the way, and collecting mime part offsets and sizes, and line counts. -func (p *Part) Walk(log *mlog.Log, parent *Part) error { +func (p *Part) Walk(elog *slog.Logger, parent *Part) error { + log := mlog.New("message", elog) + if len(p.bound) == 0 { if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") { // todo: don't read whole submessage in memory... @@ -194,11 +199,11 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error { return err } br := bytes.NewReader(buf) - mp, err := Parse(log, p.strict, br) + mp, err := Parse(log.Logger, p.strict, br) if err != nil { return fmt.Errorf("parsing embedded message: %w", err) } - if err := mp.Walk(log, nil); err != nil { + if err := mp.Walk(log.Logger, nil); err != nil { // If this is a DSN and we are not in pedantic mode, accept unexpected end of // message. This is quite common because MTA's sometimes just truncate the original // message in a place that makes the message invalid. @@ -220,14 +225,14 @@ func (p *Part) Walk(log *mlog.Log, parent *Part) error { } for { - pp, err := p.ParseNextPart(log) + pp, err := p.ParseNextPart(log.Logger) if err == io.EOF { return nil } if err != nil { return err } - if err := pp.Walk(log, p); err != nil { + if err := pp.Walk(log.Logger, p); err != nil { return err } } @@ -241,7 +246,7 @@ func (p *Part) String() string { // newPart parses a new part, which can be the top-level message. // offset is the bound offset for parts, and the start of message for top-level messages. parent indicates if this is a top-level message or sub-part. // If an error occurs, p's exported values can still be relevant. EnsurePart uses these values. -func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) { +func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Part) (p Part, rerr error) { if r == nil { panic("nil reader") } @@ -325,14 +330,14 @@ func newPart(log *mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Pa p.MediaType = "APPLICATION" p.MediaSubType = "OCTET-STREAM" } - log.Debugx("malformed content-type, attempting to recover and continuing", err, mlog.Field("contenttype", p.header.Get("Content-Type")), mlog.Field("mediatype", p.MediaType), mlog.Field("mediasubtype", p.MediaSubType)) + log.Debugx("malformed content-type, attempting to recover and continuing", err, slog.String("contenttype", p.header.Get("Content-Type")), slog.String("mediatype", p.MediaType), slog.String("mediasubtype", p.MediaSubType)) } else if mt != "" { t := strings.SplitN(strings.ToUpper(mt), "/", 2) if len(t) != 2 { if moxvar.Pedantic || strict { return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct) } - log.Debug("malformed media-type, ignoring and continuing", mlog.Field("type", mt)) + log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt)) p.MediaType = "APPLICATION" p.MediaSubType = "OCTET-STREAM" } else { @@ -444,7 +449,7 @@ var wordDecoder = mime.WordDecoder{ }, } -func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) { +func parseEnvelope(log mlog.Log, h mail.Header) (*Envelope, error) { date, _ := h.Date() // We currently marshal this field to JSON. But JSON cannot represent all @@ -478,7 +483,7 @@ func parseEnvelope(log *mlog.Log, h mail.Header) (*Envelope, error) { return env, nil } -func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address { +func parseAddressList(log mlog.Log, h mail.Header, k string) []Address { // todo: possibly work around ios mail generating incorrect q-encoded "phrases" with unencoded double quotes? ../rfc/2047:382 l, err := h.AddressList(k) if err != nil { @@ -490,7 +495,7 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address { var user, host string addr, err := smtp.ParseAddress(a.Address) if err != nil { - log.Infox("parsing address (continuing)", err, mlog.Field("address", a.Address)) + log.Infox("parsing address (continuing)", err, slog.Any("address", a.Address)) } else { user = addr.Localpart.String() host = addr.Domain.ASCII @@ -503,7 +508,9 @@ func parseAddressList(log *mlog.Log, h mail.Header, k string) []Address { // ParseNextPart parses the next (sub)part of this multipart message. // ParseNextPart returns io.EOF and a nil part when there are no more parts. // Only used for initial parsing of message. Once parsed, use p.Parts. -func (p *Part) ParseNextPart(log *mlog.Log) (*Part, error) { +func (p *Part) ParseNextPart(elog *slog.Logger) (*Part, error) { + log := mlog.New("message", elog) + if len(p.bound) == 0 { return nil, errNotMultipart } diff --git a/message/part_test.go b/message/part_test.go index 0d19ff9..6935cfd 100644 --- a/message/part_test.go +++ b/message/part_test.go @@ -15,7 +15,7 @@ import ( "github.com/mjl-/mox/moxvar" ) -var xlog = mlog.New("message") +var pkglog = mlog.New("message", nil) func tcheck(t *testing.T, err error, msg string) { t.Helper() @@ -40,7 +40,7 @@ func tfail(t *testing.T, err, expErr error) { func TestEmptyHeader(t *testing.T) { s := "\r\nx" - p, err := EnsurePart(xlog, true, strings.NewReader(s), int64(len(s))) + p, err := EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s))) tcheck(t, err, "parse empty headers") buf, err := io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -56,7 +56,7 @@ func TestBadContentType(t *testing.T) { // Pedantic is like strict. moxvar.Pedantic = true s := "content-type: text/html;;\r\n\r\ntest" - p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) + p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) tfail(t, err, ErrBadContentType) buf, err := io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -67,7 +67,7 @@ func TestBadContentType(t *testing.T) { // Strict s = "content-type: text/html;;\r\n\r\ntest" - p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s))) + p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s))) tfail(t, err, ErrBadContentType) buf, err = io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -77,7 +77,7 @@ func TestBadContentType(t *testing.T) { // Non-strict but unrecoverable content-type. s = "content-type: not a content type;;\r\n\r\ntest" - p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) + p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) tcheck(t, err, "parsing message with bad but recoverable content-type") buf, err = io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -87,7 +87,7 @@ func TestBadContentType(t *testing.T) { // We try to use only the content-type, typically better than application/octet-stream. s = "content-type: text/html;;\r\n\r\ntest" - p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) + p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) tcheck(t, err, "parsing message with bad but recoverable content-type") buf, err = io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -97,7 +97,7 @@ func TestBadContentType(t *testing.T) { // Not recovering multipart, we won't have a boundary. s = "content-type: multipart/mixed;;\r\n\r\ntest" - p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) + p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) tcheck(t, err, "parsing message with bad but recoverable content-type") buf, err = io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -112,20 +112,20 @@ func TestBareCR(t *testing.T) { // Pedantic is like strict. moxvar.Pedantic = true - p, err := EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) + p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) tfail(t, err, errBareCR) _, err = io.ReadAll(p.Reader()) tfail(t, err, errBareCR) moxvar.Pedantic = false // Strict. - p, err = EnsurePart(xlog, true, strings.NewReader(s), int64(len(s))) + p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s))) tfail(t, err, errBareCR) _, err = io.ReadAll(p.Reader()) tcheck(t, err, "read fallback part without error") // Non-strict allows bare cr. - p, err = EnsurePart(xlog, false, strings.NewReader(s), int64(len(s))) + p, err = EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) tcheck(t, err, "parse") buf, err := io.ReadAll(p.Reader()) tcheck(t, err, "read") @@ -141,7 +141,7 @@ aGkK func TestBasic(t *testing.T) { r := strings.NewReader(basicMsg) - p, err := Parse(xlog, true, r) + p, err := Parse(pkglog.Logger, true, r) tcheck(t, err, "new reader") buf, err := io.ReadAll(p.RawReader()) @@ -176,7 +176,7 @@ Hello Joe, do you think we can meet at 3:30 tomorrow? func TestBasic2(t *testing.T) { r := strings.NewReader(basicMsg2) - p, err := Parse(xlog, true, r) + p, err := Parse(pkglog.Logger, true, r) tcheck(t, err, "new reader") buf, err := io.ReadAll(p.RawReader()) @@ -196,9 +196,9 @@ func TestBasic2(t *testing.T) { } r = strings.NewReader(basicMsg2) - p, err = Parse(xlog, true, r) + p, err = Parse(pkglog.Logger, true, r) tcheck(t, err, "new reader") - err = p.Walk(xlog, nil) + err = p.Walk(pkglog.Logger, nil) tcheck(t, err, "walk") if p.RawLineCount != 2 { t.Fatalf("basic message, got %d lines, expected 2", p.RawLineCount) @@ -237,25 +237,25 @@ This is the epilogue. It is also to be ignored. func TestMime(t *testing.T) { // from ../rfc/2046:1148 r := strings.NewReader(mimeMsg) - p, err := Parse(xlog, true, r) + p, err := Parse(pkglog.Logger, true, r) tcheck(t, err, "new reader") if len(p.bound) == 0 { t.Fatalf("got no bound, expected bound for mime message") } - pp, err := p.ParseNextPart(xlog) + pp, err := p.ParseNextPart(pkglog.Logger) tcheck(t, err, "next part") buf, err := io.ReadAll(pp.Reader()) tcheck(t, err, "read all") tcompare(t, string(buf), "This is implicitly typed plain US-ASCII text.\r\nIt does NOT end with a linebreak.") - pp, err = p.ParseNextPart(xlog) + pp, err = p.ParseNextPart(pkglog.Logger) tcheck(t, err, "next part") buf, err = io.ReadAll(pp.Reader()) tcheck(t, err, "read all") tcompare(t, string(buf), "This is explicitly typed plain US-ASCII text.\r\nIt DOES end with a linebreak.\r\n") - _, err = p.ParseNextPart(xlog) + _, err = p.ParseNextPart(pkglog.Logger) tcompare(t, err, io.EOF) if len(p.Parts) != 2 { @@ -274,17 +274,17 @@ func TestLongLine(t *testing.T) { for i := range line { line[i] = 'a' } - _, err := Parse(xlog, true, bytes.NewReader(line)) + _, err := Parse(pkglog.Logger, true, bytes.NewReader(line)) tfail(t, err, errLineTooLong) } func TestBareCrLf(t *testing.T) { parse := func(strict bool, s string) error { - p, err := Parse(xlog, strict, strings.NewReader(s)) + p, err := Parse(pkglog.Logger, strict, strings.NewReader(s)) if err != nil { return err } - return p.Walk(xlog, nil) + return p.Walk(pkglog.Logger, nil) } err := parse(false, "subject: test\ntest\r\n") tfail(t, err, errBareLF) @@ -316,25 +316,25 @@ func TestMissingClosingBoundary(t *testing.T) { test `, "\n", "\r\n") - msg, err := Parse(xlog, false, strings.NewReader(message)) + msg, err := Parse(pkglog.Logger, false, strings.NewReader(message)) tcheck(t, err, "new reader") err = walkmsg(&msg) tfail(t, err, errMissingClosingBoundary) - msg, _ = Parse(xlog, false, strings.NewReader(message)) - err = msg.Walk(xlog, nil) + msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message)) + err = msg.Walk(pkglog.Logger, nil) tfail(t, err, errMissingClosingBoundary) } func TestHeaderEOF(t *testing.T) { message := "header: test" - _, err := Parse(xlog, false, strings.NewReader(message)) + _, err := Parse(pkglog.Logger, false, strings.NewReader(message)) tfail(t, err, errUnexpectedEOF) } func TestBodyEOF(t *testing.T) { message := "header: test\r\n\r\ntest" - msg, err := Parse(xlog, true, strings.NewReader(message)) + msg, err := Parse(pkglog.Logger, true, strings.NewReader(message)) tcheck(t, err, "new reader") buf, err := io.ReadAll(msg.Reader()) tcheck(t, err, "read body") @@ -365,7 +365,7 @@ test `, "\n", "\r\n") - msg, err := Parse(xlog, false, strings.NewReader(message)) + msg, err := Parse(pkglog.Logger, false, strings.NewReader(message)) tcheck(t, err, "new reader") enforceSequential = true defer func() { @@ -374,8 +374,8 @@ test err = walkmsg(&msg) tcheck(t, err, "walkmsg") - msg, _ = Parse(xlog, false, strings.NewReader(message)) - err = msg.Walk(xlog, nil) + msg, _ = Parse(pkglog.Logger, false, strings.NewReader(message)) + err = msg.Walk(pkglog.Logger, nil) tcheck(t, err, "msg.Walk") } @@ -452,7 +452,7 @@ Content-Transfer-Encoding: Quoted-printable --unique-boundary-1-- `, "\n", "\r\n") - msg, err := Parse(xlog, true, strings.NewReader(nestedMessage)) + msg, err := Parse(pkglog.Logger, true, strings.NewReader(nestedMessage)) tcheck(t, err, "new reader") enforceSequential = true defer func() { @@ -477,8 +477,8 @@ Content-Transfer-Encoding: Quoted-printable t.Fatalf("got %q, expected %q", buf, exp) } - msg, _ = Parse(xlog, false, strings.NewReader(nestedMessage)) - err = msg.Walk(xlog, nil) + msg, _ = Parse(pkglog.Logger, false, strings.NewReader(nestedMessage)) + err = msg.Walk(pkglog.Logger, nil) tcheck(t, err, "msg.Walk") } @@ -518,7 +518,7 @@ func walk(path string) error { return err } defer r.Close() - msg, err := Parse(xlog, false, r) + msg, err := Parse(pkglog.Logger, false, r) if err != nil { return err } @@ -538,7 +538,7 @@ func walkmsg(msg *Part) error { } if msg.MediaType == "MESSAGE" && (msg.MediaSubType == "RFC822" || msg.MediaSubType == "GLOBAL") { - mp, err := Parse(xlog, false, bytes.NewReader(buf)) + mp, err := Parse(pkglog.Logger, false, bytes.NewReader(buf)) if err != nil { return err } @@ -566,7 +566,7 @@ func walkmsg(msg *Part) error { } for { - pp, err := msg.ParseNextPart(xlog) + pp, err := msg.ParseNextPart(pkglog.Logger) if err == io.EOF { return nil } @@ -585,7 +585,7 @@ func TestEmbedded(t *testing.T) { tcheck(t, err, "open") fi, err := f.Stat() tcheck(t, err, "stat") - _, err = EnsurePart(xlog, false, f, fi.Size()) + _, err = EnsurePart(pkglog.Logger, false, f, fi.Size()) tcheck(t, err, "parse") } @@ -594,6 +594,6 @@ func TestEmbedded2(t *testing.T) { tcheck(t, err, "readfile") buf = bytes.ReplaceAll(buf, []byte("\n"), []byte("\r\n")) - _, err = EnsurePart(xlog, false, bytes.NewReader(buf), int64(len(buf))) + _, err = EnsurePart(pkglog.Logger, false, bytes.NewReader(buf), int64(len(buf))) tfail(t, err, nil) } diff --git a/message/referencedids_test.go b/message/referencedids_test.go index cbeab0b..9dfe611 100644 --- a/message/referencedids_test.go +++ b/message/referencedids_test.go @@ -9,7 +9,7 @@ func TestReferencedIDs(t *testing.T) { check := func(msg string, expRefs []string) { t.Helper() - p, err := Parse(xlog, true, strings.NewReader(msg)) + p, err := Parse(pkglog.Logger, true, strings.NewReader(msg)) tcheck(t, err, "parsing message") h, err := p.Header() diff --git a/metrics/http.go b/metrics/http.go index 3ecc1ce..39c6b56 100644 --- a/metrics/http.go +++ b/metrics/http.go @@ -8,14 +8,14 @@ import ( "os" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("metrics") - var ( metricHTTPClient = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -34,8 +34,8 @@ var ( // HTTPClientObserve tracks the result of an HTTP transaction in a metric, and // logs the result. -func HTTPClientObserve(ctx context.Context, pkg, method string, statusCode int, err error, start time.Time) { - log := xlog.WithContext(ctx) +func HTTPClientObserve(ctx context.Context, log mlog.Log, pkg, method string, statusCode int, err error, start time.Time) { + log = log.WithPkg("metrics") var result string switch { case err == nil: @@ -57,5 +57,5 @@ func HTTPClientObserve(ctx context.Context, pkg, method string, statusCode int, result = "error" } metricHTTPClient.WithLabelValues(pkg, method, result, fmt.Sprintf("%d", statusCode)).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("httpclient result", err, mlog.Field("pkg", pkg), mlog.Field("method", method), mlog.Field("code", statusCode), mlog.Field("duration", time.Since(start))) + log.Debugx("httpclient result", err, slog.String("pkg", pkg), slog.String("method", method), slog.Int("code", statusCode), slog.Duration("duration", time.Since(start))) } diff --git a/mlog/log.go b/mlog/log.go index 0c5be6a..ab7ba1b 100644 --- a/mlog/log.go +++ b/mlog/log.go @@ -1,30 +1,21 @@ -// Package mlog provides logging with log levels and fields. +// Package mlog providers helpers on top of slog.Logger. // -// Each log level has a function to log with and without error. -// Each such function takes a varargs list of fields (key value pairs) to log. -// Variable data should be in fields. Logging strings themselves should be -// constant, for easier log processing (e.g. building metrics based on log -// messages). +// Packages of mox that are fit or use by external code take an *slog.Logger as +// parameter for logging. Internally, and packages not intended for reuse, +// logging is done with mlog.Log. It providers convenience functions for: +// logging error values, tracing (protocol messages), uncoditional printing +// optionally exiting. // -// The log levels can be configured per originating package, e.g. smtpclient, -// imapserver. The configuration is application-global, so each Log instance -// uses the same log levels. -// -// Print* should be used for lines that always should be printed, regardless of -// configured log levels. Useful for startup logging and subcommands. -// -// Fatal* stops the program. Its log text is always printed. +// An mlog provides a handler for an mlog.Log for formatting log lines. Lines are +// logged as "logfmt" lines for "mox serve". For command-line tools, the lines are +// printed with colon-separated level, message and error, followed by +// semicolon-separated attributes. package mlog -// todo: log with source=path:linenumber? and/or stacktrace (perhaps optional) -// todo: should we turn errors logged with an context.Canceled from a level error into level info? -// todo: rethink format. perhaps simply using %#v is more useful for many types? - import ( "bytes" "context" "encoding/base64" - "errors" "fmt" "io" "os" @@ -32,91 +23,101 @@ import ( "strconv" "strings" "sync/atomic" + "time" + + "golang.org/x/exp/slog" ) +var noctx = context.Background() + +// Logfmt enabled output in logfmt, instead of output more suitable for +// command-line tools. Must be set early in a program lifecycle. var Logfmt bool -type Level int - -func (l Level) String() string { - return LevelStrings[l] -} - -var LevelStrings = map[Level]string{ - LevelPrint: "print", - LevelFatal: "fatal", - LevelError: "error", - LevelInfo: "info", - LevelDebug: "debug", - LevelTrace: "trace", - LevelTraceauth: "traceauth", - LevelTracedata: "tracedata", -} - -var Levels = map[string]Level{ - "print": LevelPrint, - "fatal": LevelFatal, - "error": LevelError, - "info": LevelInfo, - "debug": LevelDebug, - "trace": LevelTrace, - "traceauth": LevelTraceauth, - "tracedata": LevelTracedata, -} - -const ( - LevelPrint Level = 0 // Printed regardless of configured log level. - LevelFatal Level = 1 // Printed regardless of configured log level. - LevelError Level = 2 - LevelInfo Level = 3 - LevelDebug Level = 4 - LevelTrace Level = 5 - LevelTraceauth Level = 6 - LevelTracedata Level = 7 -) - // LogStringer is used when formatting field values during logging. If a value // implements it, LogString is called for the value to log. type LogStringer interface { LogString() string } -// Holds a map[string]Level, mapping a package (field pkg in logs) to a log level. -// The empty string is the default/fallback log level. -var config atomic.Value +var lowestLevel atomic.Int32 // For quick initial check. +var config atomic.Pointer[map[string]slog.Level] // For secondary complete check for match. func init() { - config.Store(map[string]Level{"": LevelError}) + SetConfig(map[string]slog.Level{"": LevelInfo}) } // SetConfig atomically sets the new log levels used by all Log instances. -func SetConfig(c map[string]Level) { - config.Store(c) -} - -// Pair is a field/value pair, for use in logged lines. -type Pair struct { - Key string - Value any -} - -// Field is a shorthand for making a Pair. -func Field(k string, v any) Pair { - return Pair{k, v} -} - -// Log is an instance potentially with its own field/value pair added to any -// logging output. -type Log struct { - fields []Pair - moreFields func() []Pair -} - -// New returns a new Log instance. Each log invocation adds field "pkg". -func New(pkg string) *Log { - return &Log{ - fields: []Pair{{"pkg", pkg}}, +func SetConfig(c map[string]slog.Level) { + lowest := c[""] + for _, l := range c { + if l < lowest { + lowest = l + } } + lowestLevel.Store(int32(lowest)) + config.Store(&c) +} + +var ( + // When the configured log level is any of the Trace levels, all protocol messages + // are printed. But protocol "data" (like an email message in the SMTP DATA + // command) is replaced with "..." unless the configured level is LevelTracedata. + // Likewise, protocol messages with authentication data (e.g. plaintext base64 + // passwords) are replaced with "***" unless the configured level is LevelTraceauth + // or LevelTracedata. + LevelTracedata = slog.LevelDebug - 8 + LevelTraceauth = slog.LevelDebug - 6 + LevelTrace = slog.LevelDebug - 4 + LevelDebug = slog.LevelDebug + LevelInfo = slog.LevelInfo + LevelError = slog.LevelError + LevelFatal = slog.LevelError + 4 // Printed regardless of configured log level. + LevelPrint = slog.LevelError + 8 // Printed regardless of configured log level. +) + +// Levelstrings map log levels to human-readable names. +var LevelStrings = map[slog.Level]string{ + LevelTracedata: "tracedata", + LevelTraceauth: "traceauth", + LevelTrace: "trace", + LevelDebug: "debug", + LevelInfo: "info", + LevelError: "error", + LevelFatal: "fatal", + LevelPrint: "print", +} + +// Levels map the human-readable log level to a level. +var Levels = map[string]slog.Level{ + "tracedata": LevelTracedata, + "traceauth": LevelTraceauth, + "trace": LevelTrace, + "debug": LevelDebug, + "info": LevelInfo, + "error": LevelError, + "fatal": LevelFatal, + "print": LevelPrint, +} + +// Log wraps an slog.Logger, providing convenience functions. +type Log struct { + *slog.Logger +} + +// New returns a Log that adds a "pkg" attribute. If logger is nil, a new +// Logger is created with a custom handler. +func New(pkg string, logger *slog.Logger) Log { + if logger == nil { + logger = slog.New(&handler{}) + } + return Log{logger}.WithPkg(pkg) +} + +// WithCid adds a attribute "cid". +// Also see WithContext. +func (l Log) WithCid(cid int64) Log { + return l.With(slog.Int64("cid", cid)) } type key string @@ -124,19 +125,13 @@ type key string // CidKey can be used with context.WithValue to store a "cid" in a context, for logging. var CidKey key = "cid" -// WithCid adds a field "cid". -// Also see WithContext. -func (l *Log) WithCid(cid int64) *Log { - return l.Fields(Pair{"cid", cid}) -} - // WithContext adds cid from context, if present. Context are often passed to // functions, especially between packages, to pass a "cid" for an operation. At the // start of a function (especially if exported) a variable "log" is often -// instantiated from a package-level variable "xlog", with WithContext for its cid. -// A *Log could be passed instead, but contexts are more pervasive. For the same +// instantiated from a package-level logger, with WithContext for its cid. +// Ideally, a Log could be passed instead, but contexts are more pervasive. For the same // reason WithContext is more common than WithCid. -func (l *Log) WithContext(ctx context.Context) *Log { +func (l Log) WithContext(ctx context.Context) Log { cidv := ctx.Value(CidKey) if cidv == nil { return l @@ -145,86 +140,200 @@ func (l *Log) WithContext(ctx context.Context) *Log { return l.WithCid(cid) } -// Field adds fields to the logger. Each logged line adds these fields. -func (l *Log) Fields(fields ...Pair) *Log { - nl := *l - nl.fields = append(fields, nl.fields...) - return &nl +// With adds attributes to to each logged line. +func (l Log) With(attrs ...slog.Attr) Log { + return Log{slog.New(l.Logger.Handler().WithAttrs(attrs))} } -// MoreFields sets a function on the logger that is called just before logging, -// to retrieve additional fields to log. -func (l *Log) MoreFields(fn func() []Pair) *Log { - nl := *l - nl.moreFields = fn - return &nl +// WithPkg ensures pkg is added as attribute to logged lines. If the handler is +// an mlog handler, pkg is only added if not already the last added package. +func (l Log) WithPkg(pkg string) Log { + h := l.Logger.Handler() + if ph, ok := h.(*handler); ok { + if len(ph.Pkgs) > 0 && ph.Pkgs[len(ph.Pkgs)-1] == pkg { + return l + } + return Log{slog.New(ph.WithPkg(pkg))} + } + return Log{slog.New(h.WithAttrs([]slog.Attr{slog.String("pkg", pkg)}))} +} + +// WithFunc sets fn to be called for additional attributes. Fn is only called +// when the line is logged. +// If the underlying handler is not an mlog.handler, this method has no effect. +// Caller must take care of preventing data races. +func (l Log) WithFunc(fn func() []slog.Attr) Log { + h := l.Logger.Handler() + if ph, ok := h.(*handler); ok { + return Log{slog.New(ph.WithFunc(fn))} + } + // Ignored for other handlers, only used internally (smtpserver, imapserver). + return l } // Check logs an error if err is not nil. Intended for logging errors that are good // to know, but would not influence program flow. -func (l *Log) Check(err error, text string, fields ...Pair) { +func (l Log) Check(err error, msg string, attrs ...slog.Attr) { if err != nil { - l.Errorx(text, err, fields...) + l.Errorx(msg, err, attrs...) } } -func (l *Log) Trace(traceLevel Level, text string) bool { - return l.logx(traceLevel, nil, text) +func errAttr(err error) slog.Attr { + return slog.Any("err", err) } -func (l *Log) Fatal(text string, fields ...Pair) { l.Fatalx(text, nil, fields...) } -func (l *Log) Fatalx(text string, err error, fields ...Pair) { - l.plog(LevelFatal, err, text, fields...) +// todo: consider taking a context parameter. it would require all code be refactored. we may want to do this if callers really depend on passing attrs through context. the mox code base does not do that. it makes all call sites more tedious, and requires passing around ctx everywhere, so consider carefully. + +func (l Log) Debug(msg string, attrs ...slog.Attr) { + l.Logger.LogAttrs(noctx, LevelDebug, msg, attrs...) +} + +func (l Log) Debugx(msg string, err error, attrs ...slog.Attr) { + if err != nil { + attrs = append([]slog.Attr{errAttr(err)}, attrs...) + } + l.Logger.LogAttrs(noctx, LevelDebug, msg, attrs...) +} + +func (l Log) Info(msg string, attrs ...slog.Attr) { + l.Logger.LogAttrs(noctx, LevelInfo, msg, attrs...) +} + +func (l Log) Infox(msg string, err error, attrs ...slog.Attr) { + if err != nil { + attrs = append([]slog.Attr{errAttr(err)}, attrs...) + } + l.Logger.LogAttrs(noctx, LevelInfo, msg, attrs...) +} + +func (l Log) Error(msg string, attrs ...slog.Attr) { + l.Logger.LogAttrs(noctx, LevelError, msg, attrs...) +} + +func (l Log) Errorx(msg string, err error, attrs ...slog.Attr) { + if err != nil { + attrs = append([]slog.Attr{errAttr(err)}, attrs...) + } + l.Logger.LogAttrs(noctx, LevelError, msg, attrs...) +} + +func (l Log) Fatal(msg string, attrs ...slog.Attr) { + l.Logger.LogAttrs(noctx, LevelFatal, msg, attrs...) os.Exit(1) } -func (l *Log) Print(text string, fields ...Pair) bool { - return l.logx(LevelPrint, nil, text, fields...) -} -func (l *Log) Printx(text string, err error, fields ...Pair) bool { - return l.logx(LevelPrint, err, text, fields...) +func (l Log) Fatalx(msg string, err error, attrs ...slog.Attr) { + if err != nil { + attrs = append([]slog.Attr{errAttr(err)}, attrs...) + } + l.Logger.LogAttrs(noctx, LevelFatal, msg, attrs...) + os.Exit(1) } -func (l *Log) Debug(text string, fields ...Pair) bool { - return l.logx(LevelDebug, nil, text, fields...) -} -func (l *Log) Debugx(text string, err error, fields ...Pair) bool { - return l.logx(LevelDebug, err, text, fields...) +func (l Log) Print(msg string, attrs ...slog.Attr) { + l.Logger.LogAttrs(noctx, LevelPrint, msg, attrs...) } -func (l *Log) Info(text string, fields ...Pair) bool { return l.logx(LevelInfo, nil, text, fields...) } -func (l *Log) Infox(text string, err error, fields ...Pair) bool { - return l.logx(LevelInfo, err, text, fields...) +func (l Log) Printx(msg string, err error, attrs ...slog.Attr) { + if err != nil { + attrs = append([]slog.Attr{errAttr(err)}, attrs...) + } + l.Logger.LogAttrs(noctx, LevelPrint, msg, attrs...) } -func (l *Log) Error(text string, fields ...Pair) bool { - return l.logx(LevelError, nil, text, fields...) -} -func (l *Log) Errorx(text string, err error, fields ...Pair) bool { - return l.logx(LevelError, err, text, fields...) -} +// Trace logs at trace/traceauth/tracedata level. +// If the active log level is any of the trace levels, the data is logged. +// If level is for tracedata, but the active level doesn't trace data, data is replaced with "...". +// If level is for traceauth, but the active level doesn't trace auth, data is replaced with "***". +func (l Log) Trace(level slog.Level, prefix string, data []byte) { + h := l.Handler() + if !h.Enabled(noctx, level) { + return + } + ph, ok := h.(*handler) + if !ok { + msg := prefix + string(data) + r := slog.NewRecord(time.Now(), level, msg, 0) + h.Handle(noctx, r) + return + } + filterLevel, ok := ph.configMatch(level) + if !ok { + return + } -func (l *Log) logx(level Level, err error, text string, fields ...Pair) bool { - if ok, high := l.match(level); ok { - // Nothing. - } else if high >= LevelTrace && level == LevelTraceauth { - text = "***" - } else if high >= LevelTrace && level == LevelTracedata { - text = "..." + var msg string + if hideData, hideAuth := traceLevel(filterLevel, level); hideData { + msg = prefix + "..." + } else if hideAuth { + msg = prefix + "***" } else { - return false + msg = prefix + string(data) } - if level > LevelTrace { - level = LevelTrace + r := slog.NewRecord(time.Time{}, level, msg, 0) + ph.write(filterLevel, r) +} + +func traceLevel(level, recordLevel slog.Level) (hideData, hideAuth bool) { + hideData = recordLevel == LevelTracedata && level > LevelTracedata + hideAuth = recordLevel == LevelTraceauth && level > LevelTraceauth + return +} + +type handler struct { + Pkgs []string + Attrs []slog.Attr + Group string // Empty or with dot-separated names, ending with a dot. + Fn func() []slog.Attr // Only called when record is actually being logged. +} + +func match(minLevel, level slog.Level) bool { + return level >= LevelFatal || level >= minLevel || minLevel <= LevelTrace && level <= LevelTrace +} + +func (h *handler) Enabled(ctx context.Context, level slog.Level) bool { + return match(slog.Level(lowestLevel.Load()), level) +} + +func (h *handler) configMatch(level slog.Level) (slog.Level, bool) { + c := *config.Load() + for i := len(h.Pkgs) - 1; i >= 0; i-- { + if l, ok := c[h.Pkgs[i]]; ok { + return l, match(l, level) + } + } + l := c[""] + return l, match(l, level) +} + +func (h *handler) Handle(ctx context.Context, r slog.Record) error { + l, ok := h.configMatch(r.Level) + if !ok { + return nil + } + if hideData, hideAuth := traceLevel(l, r.Level); hideData { + r.Message = "..." + } else if hideAuth { + r.Message = "***" + } + return h.write(l, r) +} + +// Reuse buffers to format log lines into. +var logBuffersStore [32][256]byte +var logBuffers = make(chan []byte, 200) + +func init() { + for i := range logBuffersStore { + logBuffers <- logBuffersStore[i][:] } - l.plog(level, err, text, fields...) - return true } // escape logfmt string if required, otherwise return original string. -func logfmtValue(s string) string { +func formatString(s string) string { for _, c := range s { - if c == '"' || c == '\\' || c <= ' ' || c == '=' || c >= 0x7f { + if c <= ' ' || c == '"' || c == '\\' || c == '=' || c >= 0x7f { return fmt.Sprintf("%q", s) } } @@ -261,6 +370,8 @@ func stringValue(iscid, nested bool, v any) string { return "" } return "[" + strings.Join(r, ",") + "]" + case error: + return r.Error() } rv := reflect.ValueOf(v) @@ -320,100 +431,188 @@ func stringValue(iscid, nested bool, v any) string { } first = false k := strings.ToLower(t.Field(i).Name) - b.WriteString(k + "=" + logfmtValue(vs)) + b.WriteString(k + "=" + vs) } return b.String() } -func (l *Log) plog(level Level, err error, text string, fields ...Pair) { - fields = append(l.fields, fields...) - if l.moreFields != nil { - fields = append(fields, l.moreFields()...) +func writeAttr(w io.Writer, separator, group string, a slog.Attr) { + switch a.Value.Kind() { + case slog.KindGroup: + if group != "" { + group += "." + } + group += a.Key + for _, a := range a.Value.Group() { + writeAttr(w, separator, group, a) + } + return + default: + var vv any + if a.Value.Kind() == slog.KindLogValuer { + vv = a.Value.Resolve().Any() + } else { + vv = a.Value.Any() + } + s := stringValue(a.Key == "cid", false, vv) + fmt.Fprint(w, separator, group, a.Key, "=", formatString(s)) } - // We build up a buffer so we can do a single atomic write of the data. Otherwise partial log lines may interleaf. - b := &bytes.Buffer{} - if Logfmt { - fmt.Fprintf(b, "l=%s m=%s", LevelStrings[level], logfmtValue(text)) - if err != nil { - fmt.Fprintf(b, " err=%s", logfmtValue(err.Error())) - } - for i := 0; i < len(fields); i++ { - kv := fields[i] - fmt.Fprintf(b, " %s=%s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value))) - } - b.WriteString("\n") - } else { - fmt.Fprintf(b, "%s: %s", LevelStrings[level], logfmtValue(text)) - if err != nil { - fmt.Fprintf(b, ": %s", logfmtValue(err.Error())) - } - if len(fields) > 0 { - fmt.Fprint(b, " (") - for i := 0; i < len(fields); i++ { - if i > 0 { - fmt.Fprint(b, "; ") - } - kv := fields[i] - fmt.Fprintf(b, "%s: %s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value))) - } - fmt.Fprint(b, ")") - } - b.WriteString("\n") - } - os.Stderr.Write(b.Bytes()) } -func (l *Log) match(level Level) (bool, Level) { - if level == LevelPrint || level == LevelFatal { - return true, level +func (h *handler) write(l slog.Level, r slog.Record) error { + // Reuse a buffer, or temporarily allocate a new one. + var buf []byte + select { + case buf = <-logBuffers: + defer func() { + logBuffers <- buf + }() + default: + buf = make([]byte, 128) } - cl := config.Load().(map[string]Level) + b := bytes.NewBuffer(buf[:0]) + eb := &errWriter{b, nil} - seen := false - var high Level - for _, kv := range l.fields { - if kv.Key != "pkg" { - continue + if Logfmt { + var wrotePkgs bool + ensurePkgs := func() { + if !wrotePkgs { + wrotePkgs = true + for _, pkg := range h.Pkgs { + writeAttr(eb, " ", "", slog.String("pkg", pkg)) + } + } } - pkg, ok := kv.Value.(string) - if !ok { - continue + + fmt.Fprint(eb, "l=", LevelStrings[r.Level], " m=") + fmt.Fprintf(eb, "%q", r.Message) + n := 0 + r.Attrs(func(a slog.Attr) bool { + if n > 0 || a.Key != "err" || h.Group != "" { + ensurePkgs() + } + writeAttr(eb, " ", h.Group, a) + n++ + return true + }) + ensurePkgs() + for _, a := range h.Attrs { + writeAttr(eb, " ", h.Group, a) } - v, ok := cl[pkg] - if v > high { - high = v + if h.Fn != nil { + for _, a := range h.Fn() { + writeAttr(eb, " ", h.Group, a) + } } - if ok && v >= level { - return true, high + fmt.Fprint(eb, "\n") + } else { + var wrotePkgs bool + ensurePkgs := func() { + if !wrotePkgs { + wrotePkgs = true + for _, pkg := range h.Pkgs { + writeAttr(eb, "; ", "", slog.String("pkg", pkg)) + } + } } - seen = seen || ok + + fmt.Fprint(eb, LevelStrings[r.Level], ": ", r.Message) + n := 0 + r.Attrs(func(a slog.Attr) bool { + if n == 0 && a.Key == "err" && h.Group == "" { + fmt.Fprint(eb, ": ", a.Value.String()) + ensurePkgs() + } else { + ensurePkgs() + writeAttr(eb, "; ", h.Group, a) + } + n++ + return true + }) + ensurePkgs() + for _, a := range h.Attrs { + writeAttr(eb, "; ", h.Group, a) + } + if h.Fn != nil { + for _, a := range h.Fn() { + writeAttr(eb, "; ", h.Group, a) + n++ + } + } + fmt.Fprint(eb, "\n") } - if seen { - return false, high + if eb.Err != nil { + return eb.Err } - v, ok := cl[""] - if v > high { - high = v - } - return ok && v >= level, v + + // todo: for mox serve, do writes in separate goroutine. + _, err := os.Stderr.Write(b.Bytes()) + return err } type errWriter struct { - log *Log - level Level - msg string + Writer *bytes.Buffer + Err error } func (w *errWriter) Write(buf []byte) (int, error) { - err := errors.New(strings.TrimSpace(string(buf))) - w.log.logx(w.level, err, w.msg) + if w.Err != nil { + return 0, w.Err + } + var n int + n, w.Err = w.Writer.Write(buf) + return n, w.Err +} + +func (h *handler) WithAttrs(attrs []slog.Attr) slog.Handler { + nh := *h + if h.Attrs != nil { + nh.Attrs = append([]slog.Attr{}, h.Attrs...) + } + nh.Attrs = append(nh.Attrs, attrs...) + return &nh +} + +func (h *handler) WithGroup(name string) slog.Handler { + if name == "" { + return h + } + nh := *h + nh.Group += name + "." + return &nh +} + +func (h *handler) WithPkg(pkg string) *handler { + nh := *h + if nh.Pkgs != nil { + nh.Pkgs = append([]string{}, nh.Pkgs...) + } + nh.Pkgs = append(nh.Pkgs, pkg) + return &nh +} + +func (h *handler) WithFunc(fn func() []slog.Attr) *handler { + nh := *h + nh.Fn = fn + return &nh +} + +type logWriter struct { + log Log + level slog.Level + msg string +} + +func (w logWriter) Write(buf []byte) (int, error) { + err := strings.TrimSpace(string(buf)) + w.log.LogAttrs(noctx, w.level, w.msg, slog.String("err", err)) return len(buf), nil } -// ErrWriter returns a writer that turns each write into a logging call on "log" +// LogWriter returns a writer that turns each write into a logging call on "log" // with given "level" and "msg" and the written content as an error. // Can be used for making a Go log.Logger for use in http.Server.ErrorLog. -func ErrWriter(log *Log, level Level, msg string) io.Writer { - return &errWriter{log, level, msg} +func LogWriter(log Log, level slog.Level, msg string) io.Writer { + return logWriter{log, level, msg} } diff --git a/mox-/admin.go b/mox-/admin.go index d60fc0e..30aff70 100644 --- a/mox-/admin.go +++ b/mox-/admin.go @@ -20,6 +20,7 @@ import ( "time" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/mjl-/adns" @@ -28,7 +29,6 @@ import ( "github.com/mjl-/mox/dmarc" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/junk" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mtasts" "github.com/mjl-/mox/smtp" "github.com/mjl-/mox/tlsrpt" @@ -154,7 +154,7 @@ func MakeAccountConfig(addr smtp.Address) config.Account { // MakeDomainConfig makes a new config for a domain, creating DKIM keys, using // accountName for DMARC and TLS reports. func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountName string, withMTASTS bool) (config.Domain, []string, error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) now := time.Now() year := now.Format("2006") @@ -164,7 +164,7 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN defer func() { for _, p := range paths { err := os.Remove(p) - log.Check(err, "removing path for domain config", mlog.Field("path", p)) + log.Check(err, "removing path for domain config", slog.String("path", p)) } }() @@ -180,7 +180,7 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN err := f.Close() log.Check(err, "closing file after error") err = os.Remove(path) - log.Check(err, "removing file after error", mlog.Field("path", path)) + log.Check(err, "removing file after error", slog.String("path", path)) } }() if _, err := f.Write(data); err != nil { @@ -288,10 +288,10 @@ func MakeDomainConfig(ctx context.Context, domain, hostname dns.Domain, accountN // If the account does not exist, it is created with localpart. Localpart must be // set only if the account does not yet exist. func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, localpart smtp.Localpart) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("adding domain", rerr, mlog.Field("domain", domain), mlog.Field("account", accountName), mlog.Field("localpart", localpart)) + log.Errorx("adding domain", rerr, slog.Any("domain", domain), slog.String("account", accountName), slog.Any("localpart", localpart)) } }() @@ -327,7 +327,7 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local defer func() { for _, f := range cleanupFiles { err := os.Remove(f) - log.Check(err, "cleaning up file after error", mlog.Field("path", f)) + log.Check(err, "cleaning up file after error", slog.String("path", f)) } }() @@ -356,7 +356,7 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("domain added", mlog.Field("domain", domain)) + log.Info("domain added", slog.Any("domain", domain)) cleanupFiles = nil // All good, don't cleanup. return nil } @@ -365,10 +365,10 @@ func DomainAdd(ctx context.Context, domain dns.Domain, accountName string, local // // No accounts are removed, also not when they still reference this domain. func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("removing domain", rerr, mlog.Field("domain", domain)) + log.Errorx("removing domain", rerr, slog.Any("domain", domain)) } }() @@ -418,16 +418,16 @@ func DomainRemove(ctx context.Context, domain dns.Domain) (rerr error) { err = os.Rename(src, dst) } if err != nil { - log.Errorx("renaming dkim private key file for removed domain", err, mlog.Field("src", src), mlog.Field("dst", dst)) + log.Errorx("renaming dkim private key file for removed domain", err, slog.String("src", src), slog.String("dst", dst)) } } - log.Info("domain removed", mlog.Field("domain", domain)) + log.Info("domain removed", slog.Any("domain", domain)) return nil } func WebserverConfigSet(ctx context.Context, domainRedirects map[string]string, webhandlers []config.WebHandler) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { log.Errorx("saving webserver config", rerr) @@ -680,10 +680,10 @@ func DomainRecords(domConf config.Domain, domain dns.Domain, hasDNSSEC bool) ([] // // Catchall addresses are not supported for AccountAdd. Add separately with AddressAdd. func AccountAdd(ctx context.Context, account, address string) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("adding account", rerr, mlog.Field("account", account), mlog.Field("address", address)) + log.Errorx("adding account", rerr, slog.String("account", account), slog.String("address", address)) } }() @@ -716,16 +716,16 @@ func AccountAdd(ctx context.Context, account, address string) (rerr error) { if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("account added", mlog.Field("account", account), mlog.Field("address", addr)) + log.Info("account added", slog.String("account", account), slog.Any("address", addr)) return nil } // AccountRemove removes an account and reloads the configuration. func AccountRemove(ctx context.Context, account string) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("adding account", rerr, mlog.Field("account", account)) + log.Errorx("adding account", rerr, slog.String("account", account)) } }() @@ -750,7 +750,7 @@ func AccountRemove(ctx context.Context, account string) (rerr error) { if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("account removed", mlog.Field("account", account)) + log.Info("account removed", slog.String("account", account)) return nil } @@ -775,10 +775,10 @@ func checkAddressAvailable(addr smtp.Address) error { // AddressAdd adds an email address to an account and reloads the configuration. If // address starts with an @ it is treated as a catchall address for the domain. func AddressAdd(ctx context.Context, address, account string) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("adding address", rerr, mlog.Field("address", address), mlog.Field("account", account)) + log.Errorx("adding address", rerr, slog.String("address", address), slog.String("account", account)) } }() @@ -834,16 +834,16 @@ func AddressAdd(ctx context.Context, address, account string) (rerr error) { if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("address added", mlog.Field("address", address), mlog.Field("account", account)) + log.Info("address added", slog.String("address", address), slog.String("account", account)) return nil } // AddressRemove removes an email address and reloads the configuration. func AddressRemove(ctx context.Context, address string) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("removing address", rerr, mlog.Field("address", address)) + log.Errorx("removing address", rerr, slog.String("address", address)) } }() @@ -884,16 +884,16 @@ func AddressRemove(ctx context.Context, address string) (rerr error) { if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("address removed", mlog.Field("address", address), mlog.Field("account", ad.Account)) + log.Info("address removed", slog.String("address", address), slog.String("account", ad.Account)) return nil } // AccountFullNameSave updates the full name for an account and reloads the configuration. func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("saving account full name", rerr, mlog.Field("account", account)) + log.Errorx("saving account full name", rerr, slog.String("account", account)) } }() @@ -920,16 +920,16 @@ func AccountFullNameSave(ctx context.Context, account, fullName string) (rerr er if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("account full name saved", mlog.Field("account", account)) + log.Info("account full name saved", slog.String("account", account)) return nil } // DestinationSave updates a destination for an account and reloads the configuration. func DestinationSave(ctx context.Context, account, destName string, newDest config.Destination) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("saving destination", rerr, mlog.Field("account", account), mlog.Field("destname", destName), mlog.Field("destination", newDest)) + log.Errorx("saving destination", rerr, slog.String("account", account), slog.String("destname", destName), slog.Any("destination", newDest)) } }() @@ -965,16 +965,16 @@ func DestinationSave(ctx context.Context, account, destName string, newDest conf if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("destination saved", mlog.Field("account", account), mlog.Field("destname", destName)) + log.Info("destination saved", slog.String("account", account), slog.String("destname", destName)) return nil } // AccountLimitsSave saves new message sending limits for an account. func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesPerDay, maxFirstTimeRecipientsPerDay int) (rerr error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) defer func() { if rerr != nil { - log.Errorx("saving account limits", rerr, mlog.Field("account", account)) + log.Errorx("saving account limits", rerr, slog.String("account", account)) } }() @@ -1001,7 +1001,7 @@ func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesP if err := writeDynamic(ctx, log, nc); err != nil { return fmt.Errorf("writing domains.conf: %v", err) } - log.Info("account limits saved", mlog.Field("account", account)) + log.Info("account limits saved", slog.String("account", account)) return nil } @@ -1157,7 +1157,7 @@ func ClientConfigsDomain(d dns.Domain) (ClientConfigs, error) { // IPs returns ip addresses we may be listening/receiving mail on or // connecting/sending from to the outside. func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) // Try to gather all IPs we are listening on by going through the config. // If we encounter 0.0.0.0 or ::, we'll gather all local IPs afterwards. @@ -1208,7 +1208,7 @@ func IPs(ctx context.Context, receiveOnly bool) ([]net.IP, error) { for _, addr := range addrs { ip, _, err := net.ParseCIDR(addr.String()) if err != nil { - log.Errorx("bad interface addr", err, mlog.Field("address", addr)) + log.Errorx("bad interface addr", err, slog.Any("address", addr)) continue } v4 := ip.To4() != nil diff --git a/mox-/config.go b/mox-/config.go index 4448273..26b5869 100644 --- a/mox-/config.go +++ b/mox-/config.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "golang.org/x/exp/slog" "golang.org/x/text/unicode/norm" "github.com/mjl-/autocert" @@ -44,14 +45,14 @@ import ( "github.com/mjl-/mox/smtp" ) -var xlog = mlog.New("mox") +var pkglog = mlog.New("mox", nil) // Config paths are set early in program startup. They will point to files in // the same directory. var ( ConfigStaticPath string ConfigDynamicPath string - Conf = Config{Log: map[string]mlog.Level{"": mlog.LevelError}} + Conf = Config{Log: map[string]slog.Level{"": slog.LevelError}} ) // Config as used in the code, a processed version of what is in the config file. @@ -61,7 +62,7 @@ type Config struct { Static config.Static // Does not change during the lifetime of a running instance. logMutex sync.Mutex // For accessing the log levels. - Log map[string]mlog.Level + Log map[string]slog.Level dynamicMutex sync.Mutex Dynamic config.Dynamic // Can only be accessed directly by tests. Use methods on Config for locked access. @@ -83,31 +84,31 @@ type AccountDestination struct { // LogLevelSet sets a new log level for pkg. An empty pkg sets the default log // value that is used if no explicit log level is configured for a package. // This change is ephemeral, no config file is changed. -func (c *Config) LogLevelSet(pkg string, level mlog.Level) { +func (c *Config) LogLevelSet(log mlog.Log, pkg string, level slog.Level) { c.logMutex.Lock() defer c.logMutex.Unlock() l := c.copyLogLevels() l[pkg] = level c.Log = l - xlog.Print("log level changed", mlog.Field("pkg", pkg), mlog.Field("level", mlog.LevelStrings[level])) + log.Print("log level changed", slog.String("pkg", pkg), slog.Any("level", mlog.LevelStrings[level])) mlog.SetConfig(c.Log) } // LogLevelRemove removes a configured log level for a package. -func (c *Config) LogLevelRemove(pkg string) { +func (c *Config) LogLevelRemove(log mlog.Log, pkg string) { c.logMutex.Lock() defer c.logMutex.Unlock() l := c.copyLogLevels() delete(l, pkg) c.Log = l - xlog.Print("log level cleared", mlog.Field("pkg", pkg)) + log.Print("log level cleared", slog.String("pkg", pkg)) mlog.SetConfig(c.Log) } // copyLogLevels returns a copy of c.Log, for modifications. // must be called with log lock held. -func (c *Config) copyLogLevels() map[string]mlog.Level { - m := map[string]mlog.Level{} +func (c *Config) copyLogLevels() map[string]slog.Level { + m := map[string]slog.Level{} for pkg, level := range c.Log { m[pkg] = level } @@ -115,7 +116,7 @@ func (c *Config) copyLogLevels() map[string]mlog.Level { } // LogLevels returns a copy of the current log levels. -func (c *Config) LogLevels() map[string]mlog.Level { +func (c *Config) LogLevels() map[string]slog.Level { c.logMutex.Lock() defer c.logMutex.Unlock() return c.copyLogLevels() @@ -128,12 +129,12 @@ func (c *Config) withDynamicLock(fn func()) { if now.Sub(c.DynamicLastCheck) > time.Second { c.DynamicLastCheck = now if fi, err := os.Stat(ConfigDynamicPath); err != nil { - xlog.Errorx("stat domains config", err) + pkglog.Errorx("stat domains config", err) } else if !fi.ModTime().Equal(c.dynamicMtime) { if errs := c.loadDynamic(); len(errs) > 0 { - xlog.Errorx("loading domains config", errs[0], mlog.Field("errors", errs)) + pkglog.Errorx("loading domains config", errs[0], slog.Any("errors", errs)) } else { - xlog.Info("domains config reloaded") + pkglog.Info("domains config reloaded") c.dynamicMtime = fi.ModTime() } } @@ -143,14 +144,14 @@ func (c *Config) withDynamicLock(fn func()) { // must be called with dynamic lock held. func (c *Config) loadDynamic() []error { - d, mtime, accDests, err := ParseDynamicConfig(context.Background(), ConfigDynamicPath, c.Static) + d, mtime, accDests, err := ParseDynamicConfig(context.Background(), pkglog, ConfigDynamicPath, c.Static) if err != nil { return err } c.Dynamic = d c.dynamicMtime = mtime c.accountDestinations = accDests - c.allowACMEHosts(true) + c.allowACMEHosts(pkglog, true) return nil } @@ -236,7 +237,7 @@ func (c *Config) Routes(accountName string, domain dns.Domain) (accountRoutes, d return } -func (c *Config) allowACMEHosts(checkACMEHosts bool) { +func (c *Config) allowACMEHosts(log mlog.Log, checkACMEHosts bool) { for _, l := range c.Static.Listeners { if l.TLS == nil || l.TLS.ACME == "" { continue @@ -259,7 +260,7 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) { if l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS { if d, err := dns.ParseDomain("autoconfig." + dom.Domain.ASCII); err != nil { - xlog.Errorx("parsing autoconfig domain", err, mlog.Field("domain", dom.Domain)) + log.Errorx("parsing autoconfig domain", err, slog.Any("domain", dom.Domain)) } else { hostnames[d] = struct{}{} } @@ -268,7 +269,7 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) { if l.MTASTSHTTPS.Enabled && dom.MTASTS != nil && !l.MTASTSHTTPS.NonTLS { d, err := dns.ParseDomain("mta-sts." + dom.Domain.ASCII) if err != nil { - xlog.Errorx("parsing mta-sts domain", err, mlog.Field("domain", dom.Domain)) + log.Errorx("parsing mta-sts domain", err, slog.Any("domain", dom.Domain)) } else { hostnames[d] = struct{}{} } @@ -292,15 +293,15 @@ func (c *Config) allowACMEHosts(checkACMEHosts bool) { if public.IPsNATed { ips = nil } - m.SetAllowedHostnames(dns.StrictResolver{Pkg: "autotls"}, hostnames, ips, checkACMEHosts) + m.SetAllowedHostnames(log, dns.StrictResolver{Pkg: "autotls", Log: log.Logger}, hostnames, ips, checkACMEHosts) } } // todo future: write config parsing & writing code that can read a config and remembers the exact tokens including newlines and comments, and can write back a modified file. the goal is to be able to write a config file automatically (after changing fields through the ui), but not loose comments and whitespace, to still get useful diffs for storing the config in a version control system. // must be called with lock held. -func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error { - accDests, errs := prepareDynamicConfig(ctx, ConfigDynamicPath, Conf.Static, &c) +func writeDynamic(ctx context.Context, log mlog.Log, c config.Dynamic) error { + accDests, errs := prepareDynamicConfig(ctx, log, ConfigDynamicPath, Conf.Static, &c) if len(errs) > 0 { return errs[0] } @@ -330,7 +331,7 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error { if err := f.Sync(); err != nil { return fmt.Errorf("sync domains.conf after write: %v", err) } - if err := moxio.SyncDir(filepath.Dir(ConfigDynamicPath)); err != nil { + if err := moxio.SyncDir(log, filepath.Dir(ConfigDynamicPath)); err != nil { return fmt.Errorf("sync dir of domains.conf after write: %v", err) } @@ -349,32 +350,32 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error { Conf.Dynamic = c Conf.accountDestinations = accDests - Conf.allowACMEHosts(true) + Conf.allowACMEHosts(log, true) return nil } // MustLoadConfig loads the config, quitting on errors. func MustLoadConfig(doLoadTLSKeyCerts, checkACMEHosts bool) { - errs := LoadConfig(context.Background(), doLoadTLSKeyCerts, checkACMEHosts) + errs := LoadConfig(context.Background(), pkglog, doLoadTLSKeyCerts, checkACMEHosts) if len(errs) > 1 { - xlog.Error("loading config file: multiple errors") + pkglog.Error("loading config file: multiple errors") for _, err := range errs { - xlog.Errorx("config error", err) + pkglog.Errorx("config error", err) } - xlog.Fatal("stopping after multiple config errors") + pkglog.Fatal("stopping after multiple config errors") } else if len(errs) == 1 { - xlog.Fatalx("loading config file", errs[0]) + pkglog.Fatalx("loading config file", errs[0]) } } // LoadConfig attempts to parse and load a config, returning any errors // encountered. -func LoadConfig(ctx context.Context, doLoadTLSKeyCerts, checkACMEHosts bool) []error { +func LoadConfig(ctx context.Context, log mlog.Log, doLoadTLSKeyCerts, checkACMEHosts bool) []error { Shutdown, ShutdownCancel = context.WithCancel(context.Background()) Context, ContextCancel = context.WithCancel(context.Background()) - c, errs := ParseConfig(ctx, ConfigStaticPath, false, doLoadTLSKeyCerts, checkACMEHosts) + c, errs := ParseConfig(ctx, log, ConfigStaticPath, false, doLoadTLSKeyCerts, checkACMEHosts) if len(errs) > 0 { return errs } @@ -405,7 +406,7 @@ func SetConfig(c *Config) { // quickstart in the case the user is going to provide their own certificates. // If checkACMEHosts is true, the hosts allowed for acme are compared with the // explicitly configured ips we are listening on. -func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, checkACMEHosts bool) (c *Config, errs []error) { +func ParseConfig(ctx context.Context, log mlog.Log, p string, checkOnly, doLoadTLSKeyCerts, checkACMEHosts bool) (c *Config, errs []error) { c = &Config{ Static: config.Static{ DataDir: ".", @@ -424,15 +425,15 @@ func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, ch return nil, []error{fmt.Errorf("parsing %s%v", p, err)} } - if xerrs := PrepareStaticConfig(ctx, p, c, checkOnly, doLoadTLSKeyCerts); len(xerrs) > 0 { + if xerrs := PrepareStaticConfig(ctx, log, p, c, checkOnly, doLoadTLSKeyCerts); len(xerrs) > 0 { return nil, xerrs } pp := filepath.Join(filepath.Dir(p), "domains.conf") - c.Dynamic, c.dynamicMtime, c.accountDestinations, errs = ParseDynamicConfig(ctx, pp, c.Static) + c.Dynamic, c.dynamicMtime, c.accountDestinations, errs = ParseDynamicConfig(ctx, log, pp, c.Static) if !checkOnly { - c.allowACMEHosts(checkACMEHosts) + c.allowACMEHosts(log, checkACMEHosts) } return c, errs @@ -441,13 +442,11 @@ func ParseConfig(ctx context.Context, p string, checkOnly, doLoadTLSKeyCerts, ch // PrepareStaticConfig parses the static config file and prepares data structures // for starting mox. If checkOnly is set no substantial changes are made, like // creating an ACME registration. -func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, checkOnly, doLoadTLSKeyCerts bool) (errs []error) { +func PrepareStaticConfig(ctx context.Context, log mlog.Log, configFile string, conf *Config, checkOnly, doLoadTLSKeyCerts bool) (errs []error) { addErrorf := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) } - log := xlog.WithContext(ctx) - c := &conf.Static // check that mailbox is in unicode NFC normalized form. @@ -461,7 +460,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c // Post-process logging config. if logLevel, ok := mlog.Levels[c.LogLevel]; ok { - conf.Log = map[string]mlog.Level{"": logLevel} + conf.Log = map[string]slog.Level{"": logLevel} } else { addErrorf("invalid log level %q", c.LogLevel) } @@ -569,10 +568,10 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c key = findACMEHostPrivateKey(acmeName, host, keyType, 2) } if key != nil { - log.Debug("found existing private key for certificate for host", mlog.Field("acmename", acmeName), mlog.Field("host", host), mlog.Field("keytype", keyType)) + log.Debug("found existing private key for certificate for host", slog.String("acmename", acmeName), slog.String("host", host), slog.Any("keytype", keyType)) return key, nil } - log.Debug("generating new private key for certificate for host", mlog.Field("acmename", acmeName), mlog.Field("host", host), mlog.Field("keytype", keyType)) + log.Debug("generating new private key for certificate for host", slog.String("acmename", acmeName), slog.String("host", host), slog.Any("keytype", keyType)) switch keyType { case autocert.KeyRSA2048: return rsa.GenerateKey(cryptorand.Reader, 2048) @@ -658,18 +657,18 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c switch k := privKey.(type) { case *rsa.PrivateKey: if k.N.BitLen() != 2048 { - log.Error("need rsa key with 2048 bits, for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath), mlog.Field("bits", k.N.BitLen())) + log.Error("need rsa key with 2048 bits, for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath), slog.Int("bits", k.N.BitLen())) continue } l.TLS.HostPrivateRSA2048Keys = append(l.TLS.HostPrivateRSA2048Keys, k) case *ecdsa.PrivateKey: if k.Curve != elliptic.P256() { - log.Error("unrecognized ecdsa curve for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath)) + log.Error("unrecognized ecdsa curve for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath)) continue } l.TLS.HostPrivateECDSAP256Keys = append(l.TLS.HostPrivateECDSAP256Keys, k) default: - log.Error("unrecognized key type for host private key for DANE/ACME certificates, ignoring", mlog.Field("listener", name), mlog.Field("file", keyPath), mlog.Field("keytype", fmt.Sprintf("%T", privKey))) + log.Error("unrecognized key type for host private key for DANE/ACME certificates, ignoring", slog.String("listener", name), slog.String("file", keyPath), slog.String("keytype", fmt.Sprintf("%T", privKey))) continue } } @@ -914,7 +913,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, conf *Config, c } // PrepareDynamicConfig parses the dynamic config file given a static file. -func ParseDynamicConfig(ctx context.Context, dynamicPath string, static config.Static) (c config.Dynamic, mtime time.Time, accDests map[string]AccountDestination, errs []error) { +func ParseDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, static config.Static) (c config.Dynamic, mtime time.Time, accDests map[string]AccountDestination, errs []error) { addErrorf := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) } @@ -934,13 +933,11 @@ func ParseDynamicConfig(ctx context.Context, dynamicPath string, static config.S return } - accDests, errs = prepareDynamicConfig(ctx, dynamicPath, static, &c) + accDests, errs = prepareDynamicConfig(ctx, log, dynamicPath, static, &c) return c, fi.ModTime(), accDests, errs } -func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config.Static, c *config.Dynamic) (accDests map[string]AccountDestination, errs []error) { - log := xlog.WithContext(ctx) - +func prepareDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, static config.Static, c *config.Dynamic) (accDests map[string]AccountDestination, errs []error) { addErrorf := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) } @@ -1321,7 +1318,7 @@ func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config if !ok { addErrorf("could not find localpart %q to replace with address in destinations", lp) } else { - log.Error(`deprecation warning: support for account destination addresses specified as just localpart ("username") instead of full email address will be removed in the future; update domains.conf, for each Account, for each Destination, ensure each key is an email address by appending "@" and the default domain for the account`, mlog.Field("localpart", lp), mlog.Field("address", addr), mlog.Field("account", accName)) + log.Error(`deprecation warning: support for account destination addresses specified as just localpart ("username") instead of full email address will be removed in the future; update domains.conf, for each Account, for each Destination, ensure each key is an email address by appending "@" and the default domain for the account`, slog.Any("localpart", lp), slog.Any("address", addr), slog.String("account", accName)) acc.Destinations[addr] = dest delete(acc.Destinations, lp) } diff --git a/mox-/forkexec_unix.go b/mox-/forkexec_unix.go index 30540b2..687f7ff 100644 --- a/mox-/forkexec_unix.go +++ b/mox-/forkexec_unix.go @@ -8,7 +8,7 @@ import ( "strings" "syscall" - "github.com/mjl-/mox/mlog" + "golang.org/x/exp/slog" ) // Fork and exec as unprivileged user. @@ -19,7 +19,7 @@ import ( func ForkExecUnprivileged() { prog, err := os.Executable() if err != nil { - xlog.Fatalx("finding executable for exec", err) + pkglog.Fatalx("finding executable for exec", err) } files := []*os.File{os.Stdin, os.Stdout, os.Stderr} @@ -49,7 +49,7 @@ func ForkExecUnprivileged() { }, }) if err != nil { - xlog.Fatalx("fork and exec", err) + pkglog.Fatalx("fork and exec", err) } CleanupPassedFiles() @@ -66,9 +66,9 @@ func ForkExecUnprivileged() { st, err := p.Wait() if err != nil { - xlog.Fatalx("wait", err) + pkglog.Fatalx("wait", err) } code := st.ExitCode() - xlog.Print("stopping after child exit", mlog.Field("exitcode", code)) + pkglog.Print("stopping after child exit", slog.Int("exitcode", code)) os.Exit(code) } diff --git a/mox-/lifecycle.go b/mox-/lifecycle.go index 79a4fca..752cc18 100644 --- a/mox-/lifecycle.go +++ b/mox-/lifecycle.go @@ -32,7 +32,7 @@ func RestorePassedFiles() { if runtime.GOOS == "linux" { linuxhint = " If you updated from v0.0.1, update the mox.service file to start as root (privileges are dropped): ./mox config printservice >mox.service && sudo systemctl daemon-reload && sudo systemctl restart mox." } - xlog.Fatal("mox must be started as root, and will drop privileges after binding required sockets (missing environment variable MOX_SOCKETS)." + linuxhint) + pkglog.Fatal("mox must be started as root, and will drop privileges after binding required sockets (missing environment variable MOX_SOCKETS)." + linuxhint) } // 0,1,2 are stdin,stdout,stderr, 3 is the first passed fd (first listeners, then files). @@ -59,12 +59,12 @@ func RestorePassedFiles() { func CleanupPassedFiles() { for _, f := range passedListeners { err := f.Close() - xlog.Check(err, "closing listener socket file descriptor") + pkglog.Check(err, "closing listener socket file descriptor") } for _, fl := range passedFiles { for _, f := range fl { err := f.Close() - xlog.Check(err, "closing path file descriptor") + pkglog.Check(err, "closing path file descriptor") } } } @@ -193,7 +193,7 @@ func (c *connections) Register(nc net.Conn, protocol, listener string) { // doesn't hurt to log it. select { case <-Shutdown.Done(): - xlog.Error("new connection added while shutting down") + pkglog.Error("new connection added while shutting down") debug.PrintStack() default: } @@ -258,7 +258,7 @@ func (c *connections) Shutdown() { defer c.Unlock() for nc := range c.conns { if err := nc.SetDeadline(now); err != nil { - xlog.Errorx("setting immediate read/write deadline for shutdown", err) + pkglog.Errorx("setting immediate read/write deadline for shutdown", err) } } } diff --git a/message/tlsrecv.go b/mox-/tlsrecv.go similarity index 87% rename from message/tlsrecv.go rename to mox-/tlsrecv.go index 0193a7b..dab0879 100644 --- a/message/tlsrecv.go +++ b/mox-/tlsrecv.go @@ -1,14 +1,16 @@ -package message +package mox import ( "crypto/tls" "fmt" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/mlog" ) // TLSReceivedComment returns a comment about TLS of the connection for use in a Receive header. -func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string { +func TLSReceivedComment(log mlog.Log, cs tls.ConnectionState) []string { // todo future: we could use the "tls" clause for the Received header as specified in ../rfc/8314:496. however, the text implies it is only for submission, not regular smtp. and it cannot specify the tls version. for now, not worth the trouble. // Comments from other mail servers: @@ -32,7 +34,7 @@ func TLSReceivedComment(log *mlog.Log, cs tls.ConnectionState) []string { if version, ok := versions[cs.Version]; ok { add(version) } else { - log.Info("unknown tls version identifier", mlog.Field("version", cs.Version)) + log.Info("unknown tls version identifier", slog.Any("version", cs.Version)) add(fmt.Sprintf("TLS identifier %x", cs.Version)) } diff --git a/moxio/bufpool.go b/moxio/bufpool.go index bc2f241..32acd16 100644 --- a/moxio/bufpool.go +++ b/moxio/bufpool.go @@ -6,11 +6,11 @@ import ( "fmt" "io" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("moxio") - // todo: instead of a bufpool, should maybe just make an alternative to bufio.Reader with a big enough buffer that we can fully use to read a line. var ErrLineTooLong = errors.New("line from remote too long") // Returned by Bufpool.Readline. @@ -49,9 +49,9 @@ func (b *Bufpool) get() []byte { // be all the bytes that have been read in the buffer. If the pool is full, the // buffer is discarded, and will be cleaned up by the garbage collector. // The caller should no longer reference "buf" after a call to put. -func (b *Bufpool) put(buf []byte, n int) { +func (b *Bufpool) put(log mlog.Log, buf []byte, n int) { if len(buf) != b.size { - xlog.Error("buffer with bad size returned, ignoring", mlog.Field("badsize", len(buf)), mlog.Field("expsize", b.size)) + log.Error("buffer with bad size returned, ignoring", slog.Int("badsize", len(buf)), slog.Int("expsize", b.size)) return } @@ -67,11 +67,11 @@ func (b *Bufpool) put(buf []byte, n int) { // Readline reads a \n- or \r\n-terminated line. Line is returned without \n or \r\n. // If the line was too long, ErrLineTooLong is returned. // If an EOF is encountered before a \n, io.ErrUnexpectedEOF is returned. -func (b *Bufpool) Readline(r *bufio.Reader) (line string, rerr error) { +func (b *Bufpool) Readline(log mlog.Log, r *bufio.Reader) (line string, rerr error) { var nread int buf := b.get() defer func() { - b.put(buf, nread) + b.put(log, buf, nread) }() // Read until newline. If we reach the end of the buffer first, we write back an diff --git a/moxio/bufpool_test.go b/moxio/bufpool_test.go index f2e6d61..ce705af 100644 --- a/moxio/bufpool_test.go +++ b/moxio/bufpool_test.go @@ -7,6 +7,8 @@ import ( "io" "strings" "testing" + + "github.com/mjl-/mox/mlog" ) func TestBufpool(t *testing.T) { @@ -16,8 +18,9 @@ func TestBufpool(t *testing.T) { for i := 0; i < len(a); i++ { a[i] = 1 } - bp.put(a, len(a)) // Will be stored. - bp.put(b, 0) // Will be discarded. + log := mlog.New("moxio", nil) + bp.put(log, a, len(a)) // Will be stored. + bp.put(log, b, 0) // Will be discarded. na := bp.get() if fmt.Sprintf("%p", a) != fmt.Sprintf("%p", na) { t.Fatalf("received unexpected new buf %p != %p", a, na) @@ -28,22 +31,22 @@ func TestBufpool(t *testing.T) { } } - if _, err := bp.Readline(bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) { + if _, err := bp.Readline(log, bufio.NewReader(strings.NewReader("this is too long"))); !errors.Is(err, ErrLineTooLong) { t.Fatalf("expected ErrLineTooLong, got error %v", err) } - if _, err := bp.Readline(bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) { + if _, err := bp.Readline(log, bufio.NewReader(strings.NewReader("short"))); !errors.Is(err, io.ErrUnexpectedEOF) { t.Fatalf("expected ErrLineTooLong, got error %v", err) } er := errReader{fmt.Errorf("bad")} - if _, err := bp.Readline(bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) { + if _, err := bp.Readline(log, bufio.NewReader(er)); err == nil || !errors.Is(err, er.err) { t.Fatalf("got unexpected error %s", err) } - if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" { + if line, err := bp.Readline(log, bufio.NewReader(strings.NewReader("ok\r\n"))); line != "ok" { t.Fatalf(`got %q, err %v, expected line "ok"`, line, err) } - if line, err := bp.Readline(bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" { + if line, err := bp.Readline(log, bufio.NewReader(strings.NewReader("ok\n"))); line != "ok" { t.Fatalf(`got %q, err %v, expected line "ok"`, line, err) } } diff --git a/moxio/linkcopy.go b/moxio/linkcopy.go index 278ed82..9e8753a 100644 --- a/moxio/linkcopy.go +++ b/moxio/linkcopy.go @@ -5,6 +5,8 @@ import ( "io" "os" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/mlog" ) @@ -14,7 +16,7 @@ import ( // ensure the file is written on disk. Callers should also sync the directory of // the destination file, but may want to do that after linking/copying multiple // files. If dst was created and an error occurred, it is removed. -func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync bool) (rerr error) { +func LinkOrCopy(log mlog.Log, dst, src string, srcReaderOpt io.Reader, sync bool) (rerr error) { // Try hardlink first. err := os.Link(src, dst) if err == nil { @@ -48,7 +50,7 @@ func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync boo err := df.Close() log.Check(err, "closing partial destination file") err = os.Remove(dst) - log.Check(err, "removing partial destination file", mlog.Field("path", dst)) + log.Check(err, "removing partial destination file", slog.String("path", dst)) } }() @@ -64,7 +66,7 @@ func LinkOrCopy(log *mlog.Log, dst, src string, srcReaderOpt io.Reader, sync boo df = nil if err != nil { err := os.Remove(dst) - log.Check(err, "removing partial destination file", mlog.Field("path", dst)) + log.Check(err, "removing partial destination file", slog.String("path", dst)) return err } return nil diff --git a/moxio/linkcopy_test.go b/moxio/linkcopy_test.go index 321f86e..3b26f82 100644 --- a/moxio/linkcopy_test.go +++ b/moxio/linkcopy_test.go @@ -17,7 +17,7 @@ func tcheckf(t *testing.T, err error, format string, args ...any) { } func TestLinkOrCopy(t *testing.T) { - log := mlog.New("linkorcopy") + log := mlog.New("linkorcopy", nil) // link in same directory. file exists error. link to file in non-existent // directory (exists error). link to file in system temp dir (hopefully other file diff --git a/moxio/syncdir.go b/moxio/syncdir.go index ad3f2dd..8d9e036 100644 --- a/moxio/syncdir.go +++ b/moxio/syncdir.go @@ -5,16 +5,18 @@ package moxio import ( "fmt" "os" + + "github.com/mjl-/mox/mlog" ) // SyncDir opens a directory and syncs its contents to disk. -func SyncDir(dir string) error { +func SyncDir(log mlog.Log, dir string) error { d, err := os.Open(dir) if err != nil { return fmt.Errorf("open directory: %v", err) } err = d.Sync() xerr := d.Close() - xlog.Check(xerr, "closing directory after sync") + log.Check(xerr, "closing directory after sync") return err } diff --git a/moxio/trace.go b/moxio/trace.go index 7bb6941..a4890ec 100644 --- a/moxio/trace.go +++ b/moxio/trace.go @@ -3,43 +3,45 @@ package moxio import ( "io" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/mlog" ) type TraceWriter struct { - log *mlog.Log + log mlog.Log prefix string w io.Writer - level mlog.Level + level slog.Level } // NewTraceWriter wraps "w" into a writer that logs all writes to "log" with // log level trace, prefixed with "prefix". -func NewTraceWriter(log *mlog.Log, prefix string, w io.Writer) *TraceWriter { +func NewTraceWriter(log mlog.Log, prefix string, w io.Writer) *TraceWriter { return &TraceWriter{log, prefix, w, mlog.LevelTrace} } // Write logs a trace line for writing buf to the client, then writes to the // client. func (w *TraceWriter) Write(buf []byte) (int, error) { - w.log.Trace(w.level, w.prefix+string(buf)) + w.log.Trace(w.level, w.prefix, buf) return w.w.Write(buf) } -func (w *TraceWriter) SetTrace(level mlog.Level) { +func (w *TraceWriter) SetTrace(level slog.Level) { w.level = level } type TraceReader struct { - log *mlog.Log + log mlog.Log prefix string r io.Reader - level mlog.Level + level slog.Level } // NewTraceReader wraps reader "r" into a reader that logs all reads to "log" // with log level trace, prefixed with "prefix". -func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) *TraceReader { +func NewTraceReader(log mlog.Log, prefix string, r io.Reader) *TraceReader { return &TraceReader{log, prefix, r, mlog.LevelTrace} } @@ -48,11 +50,11 @@ func NewTraceReader(log *mlog.Log, prefix string, r io.Reader) *TraceReader { func (r *TraceReader) Read(buf []byte) (int, error) { n, err := r.r.Read(buf) if n > 0 { - r.log.Trace(r.level, r.prefix+string(buf[:n])) + r.log.Trace(r.level, r.prefix, buf[:n]) } return n, err } -func (r *TraceReader) SetTrace(level mlog.Level) { +func (r *TraceReader) SetTrace(level slog.Level) { r.level = level } diff --git a/mtasts/mtasts.go b/mtasts/mtasts.go index e47e713..4b7ff68 100644 --- a/mtasts/mtasts.go +++ b/mtasts/mtasts.go @@ -20,6 +20,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -31,8 +33,6 @@ import ( "github.com/mjl-/mox/moxio" ) -var xlog = mlog.New("mtasts") - var ( metricGet = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -190,11 +190,11 @@ var ( // LookupRecord looks up the MTA-STS TXT DNS record at "_mta-sts.", // following CNAME records, and returns the parsed MTA-STS record and the DNS TXT // record. -func LookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) { - log := xlog.WithContext(ctx) +func LookupRecord(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) { + log := mlog.New("mtasts", elog) start := time.Now() defer func() { - log.Debugx("mtasts lookup result", rerr, mlog.Field("domain", domain), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) + log.Debugx("mtasts lookup result", rerr, slog.Any("domain", domain), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start))) }() // ../rfc/8461:289 @@ -261,11 +261,11 @@ var HTTPClient = &http.Client{ // // If an error is returned, callers should back off for 5 minutes until the next // attempt. -func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policyText string, rerr error) { - log := xlog.WithContext(ctx) +func FetchPolicy(ctx context.Context, elog *slog.Logger, domain dns.Domain) (policy *Policy, policyText string, rerr error) { + log := mlog.New("mtasts", elog) start := time.Now() defer func() { - log.Debugx("mtasts fetch policy result", rerr, mlog.Field("domain", domain), mlog.Field("policy", policy), mlog.Field("policytext", policyText), mlog.Field("duration", time.Since(start))) + log.Debugx("mtasts fetch policy result", rerr, slog.Any("domain", domain), slog.Any("policy", policy), slog.String("policytext", policyText), slog.Duration("duration", time.Since(start))) }() // Timeout of 1 minute. ../rfc/8461:569 @@ -291,7 +291,7 @@ func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policy // We pass along underlying TLS certificate errors. return nil, "", fmt.Errorf("%w: http get: %w", ErrPolicyFetch, err) } - metrics.HTTPClientObserve(ctx, "mtasts", req.Method, resp.StatusCode, err, start) + metrics.HTTPClientObserve(ctx, log, "mtasts", req.Method, resp.StatusCode, err, start) defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return nil, "", ErrNoPolicy @@ -329,22 +329,22 @@ func FetchPolicy(ctx context.Context, domain dns.Domain) (policy *Policy, policy // record is still returned. // // Also see Get in package mtastsdb. -func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (record *Record, policy *Policy, policyText string, err error) { - log := xlog.WithContext(ctx) +func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (record *Record, policy *Policy, policyText string, err error) { + log := mlog.New("mtasts", elog) start := time.Now() result := "lookuperror" defer func() { metricGet.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("mtasts get result", err, mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("policy", policy), mlog.Field("duration", time.Since(start))) + log.Debugx("mtasts get result", err, slog.Any("domain", domain), slog.Any("record", record), slog.Any("policy", policy), slog.Duration("duration", time.Since(start))) }() - record, _, err = LookupRecord(ctx, resolver, domain) + record, _, err = LookupRecord(ctx, log.Logger, resolver, domain) if err != nil { return nil, nil, "", err } result = "fetcherror" - policy, policyText, err = FetchPolicy(ctx, domain) + policy, policyText, err = FetchPolicy(ctx, log.Logger, domain) if err != nil { return record, nil, "", err } diff --git a/mtasts/mtasts_test.go b/mtasts/mtasts_test.go index 1a51ef0..ed45a34 100644 --- a/mtasts/mtasts_test.go +++ b/mtasts/mtasts_test.go @@ -8,7 +8,7 @@ import ( "crypto/x509" "errors" "io" - "log" + golog "log" "math/big" "net" "net/http" @@ -18,6 +18,8 @@ import ( "testing" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/adns" "github.com/mjl-/mox/dns" @@ -25,7 +27,8 @@ import ( ) func TestLookup(t *testing.T) { - mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) + mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug}) + log := mlog.New("mtasts", nil) resolver := dns.MockResolver{ TXT: map[string][]string{ @@ -50,7 +53,7 @@ func TestLookup(t *testing.T) { test := func(host string, expRecord *Record, expErr error) { t.Helper() - record, _, err := LookupRecord(context.Background(), resolver, dns.Domain{ASCII: host}) + record, _, err := LookupRecord(context.Background(), log.Logger, resolver, dns.Domain{ASCII: host}) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("lookup: got err %#v, expected %#v", err, expErr) } @@ -184,6 +187,8 @@ func fakeCert(t *testing.T, expired bool) tls.Certificate { } func TestFetch(t *testing.T) { + log := mlog.New("mtasts", nil) + certok := fakeCert(t, false) certbad := fakeCert(t, true) @@ -218,7 +223,7 @@ func TestFetch(t *testing.T) { TLSConfig: &tls.Config{ Certificates: []tls.Certificate{cert}, }, - ErrorLog: log.New(io.Discard, "", 0), + ErrorLog: golog.New(io.Discard, "", 0), } s.ServeTLS(l, "", "") }() @@ -235,7 +240,7 @@ func TestFetch(t *testing.T) { }, } - p, _, err := FetchPolicy(context.Background(), dns.Domain{ASCII: domain}) + p, _, err := FetchPolicy(context.Background(), log.Logger, dns.Domain{ASCII: domain}) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("policy: got err %#v, expected %#v", err, expErr) } @@ -247,7 +252,7 @@ func TestFetch(t *testing.T) { expErr = ErrNoRecord } - _, p, _, err = Get(context.Background(), resolver, dns.Domain{ASCII: domain}) + _, p, _, err = Get(context.Background(), log.Logger, resolver, dns.Domain{ASCII: domain}) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("get: got err %#v, expected %#v", err, expErr) } diff --git a/mtastsdb/db.go b/mtastsdb/db.go index 5c87e0c..516e642 100644 --- a/mtastsdb/db.go +++ b/mtastsdb/db.go @@ -16,6 +16,8 @@ import ( "sync" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -28,8 +30,6 @@ import ( "github.com/mjl-/mox/tlsrpt" ) -var xlog = mlog.New("mtastsdb") - var ( metricGet = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -108,7 +108,7 @@ func Close() { defer mutex.Unlock() if DB != nil { err := DB.Close() - xlog.Check(err, "closing database") + mlog.New("mtastsdb", nil).Check(err, "closing database") DB = nil } } @@ -119,8 +119,7 @@ func Close() { // // Returns ErrNotFound if record is not present. // Returns ErrBackoff if a recent attempt to fetch a record failed. -func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) { - log := xlog.WithContext(ctx) +func lookup(ctx context.Context, log mlog.Log, domain dns.Domain) (*PolicyRecord, error) { db, err := database(ctx) if err != nil { return nil, err @@ -222,8 +221,8 @@ func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) { // Get returns an "sts" or "no-policy-found" in reportResult in most cases (when // not a local/internal error). It may add an "sts" result without policy contents // ("policy-string") in case of errors while fetching the policy. -func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) { - log := xlog.WithContext(ctx) +func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (policy *mtasts.Policy, reportResult tlsrpt.Result, fresh bool, err error) { + log := mlog.New("mtastsdb", elog) defer func() { result := "ok" if err != nil && errors.Is(err, ErrBackoff) { @@ -234,16 +233,16 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy result = "error" } metricGet.WithLabelValues(result).Inc() - log.Debugx("mtastsdb get result", err, mlog.Field("domain", domain), mlog.Field("fresh", fresh)) + log.Debugx("mtastsdb get result", err, slog.Any("domain", domain), slog.Bool("fresh", fresh)) }() - cachedPolicy, err := lookup(ctx, domain) + cachedPolicy, err := lookup(ctx, log, domain) if err != nil && errors.Is(err, ErrNotFound) { // We don't have a policy for this domain, not even a record that we tried recently // and should backoff. So attempt to fetch policy. nctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() - record, p, ptext, err := mtasts.Get(nctx, resolver, domain) + record, p, ptext, err := mtasts.Get(nctx, log.Logger, resolver, domain) if err != nil { switch { case errors.Is(err, mtasts.ErrNoRecord) || errors.Is(err, mtasts.ErrMultipleRecords) || errors.Is(err, mtasts.ErrRecordSyntax) || errors.Is(err, mtasts.ErrNoPolicy) || errors.Is(err, mtasts.ErrPolicyFetch) || errors.Is(err, mtasts.ErrPolicySyntax): @@ -303,7 +302,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy policy = &cachedPolicy.Policy nctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - record, _, err := mtasts.LookupRecord(nctx, resolver, domain) + record, _, err := mtasts.LookupRecord(nctx, log.Logger, resolver, domain) if err != nil { if errors.Is(err, mtasts.ErrNoRecord) { if policy.Mode != mtasts.ModeNone { @@ -336,7 +335,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy // didn't store the raw policy lines in the past. nctx, cancel = context.WithTimeout(ctx, 30*time.Second) defer cancel() - p, ptext, err := mtasts.FetchPolicy(nctx, domain) + p, ptext, err := mtasts.FetchPolicy(nctx, log.Logger, domain) if err != nil { log.Errorx("fetching updated policy for domain, continuing with previously cached policy", err) diff --git a/mtastsdb/db_test.go b/mtastsdb/db_test.go index 8fb32b9..11d580c 100644 --- a/mtastsdb/db_test.go +++ b/mtastsdb/db_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mtasts" ) @@ -32,6 +33,8 @@ func TestDB(t *testing.T) { os.Remove(dbpath) defer os.Remove(dbpath) + log := mlog.New("mtastsdb", nil) + if err := Init(false); err != nil { t.Fatalf("init database: %s", err) } @@ -42,7 +45,7 @@ func TestDB(t *testing.T) { timeNow = func() time.Time { return now } defer func() { timeNow = time.Now }() - if p, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != ErrNotFound { + if p, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != ErrNotFound { t.Fatalf("expected not found, got %v, %#v", err, p) } @@ -59,7 +62,7 @@ func TestDB(t *testing.T) { if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "123", &policy1, policy1.String()); err != nil { t.Fatalf("upsert record: %s", err) } - if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil { + if got, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != nil { t.Fatalf("lookup after insert: %s", err) } else if !reflect.DeepEqual(got.Policy, policy1) { t.Fatalf("mismatch between inserted and retrieved: got %#v, want %#v", got, policy1) @@ -76,7 +79,7 @@ func TestDB(t *testing.T) { if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "124", &policy2, policy2.String()); err != nil { t.Fatalf("upsert record: %s", err) } - if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil { + if got, err := lookup(ctxbg, log, dns.Domain{ASCII: "example.com"}); err != nil { t.Fatalf("lookup after insert: %s", err) } else if !reflect.DeepEqual(got.Policy, policy2) { t.Fatalf("mismatch between inserted and retrieved: got %v, want %v", got, policy2) @@ -108,7 +111,7 @@ func TestDB(t *testing.T) { t.Fatalf("records mismatch, got %#v, expected %#v", records, expRecords) } - if _, err := lookup(ctxbg, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff { + if _, err := lookup(ctxbg, log, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff { t.Fatalf("got %#v, expected ErrBackoff", err) } @@ -125,7 +128,7 @@ func TestDB(t *testing.T) { testGet := func(domain string, expPolicy *mtasts.Policy, expFresh bool, expErr error) { t.Helper() - p, _, fresh, err := Get(ctxbg, resolver, dns.Domain{ASCII: domain}) + p, _, fresh, err := Get(ctxbg, log.Logger, resolver, dns.Domain{ASCII: domain}) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("got err %v, expected %v", err, expErr) } diff --git a/mtastsdb/refresh.go b/mtastsdb/refresh.go index b321cc9..46c2d25 100644 --- a/mtastsdb/refresh.go +++ b/mtastsdb/refresh.go @@ -8,6 +8,8 @@ import ( "runtime/debug" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/dns" @@ -28,11 +30,9 @@ func refresh() int { for { ticker.Reset(interval) - ctx := context.WithValue(mox.Context, mlog.CidKey, mox.Cid()) - n, err := refresh1(ctx, dns.StrictResolver{Pkg: "mtastsdb"}, time.Sleep) - if err != nil { - xlog.WithContext(ctx).Errorx("periodic refresh of cached mtasts policies", err) - } + log := mlog.New("mtastsdb", nil).WithCid(mox.Cid()) + n, err := refresh1(mox.Context, log, dns.StrictResolver{Pkg: "mtastsdb"}, time.Sleep) + log.Check(err, "periodic refresh of cached mtasts policies") if n > 0 { refreshed += n } @@ -51,7 +51,7 @@ func refresh() int { // refreshes evenly over the next 3 hours, randomizing the domains, and we add some // jitter to the timing. Each refresh is done in a new goroutine, so a single slow // refresh doesn't mess up the timing. -func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) { +func refresh1(ctx context.Context, log mlog.Log, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) { db, err := database(ctx) if err != nil { return 0, err @@ -87,10 +87,10 @@ func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Dura } // Launch goroutine with the refresh. - xlog.WithContext(ctx).Debug("will refresh mta-sts policies over next 3 hours", mlog.Field("count", len(prs))) + log.Debug("will refresh mta-sts policies over next 3 hours", slog.Int("count", len(prs))) start := timeNow() for i, pr := range prs { - go refreshDomain(ctx, db, resolver, pr) + go refreshDomain(ctx, log, db, resolver, pr) if i < len(prs)-1 { interval := 3 * int64(time.Hour) / int64(len(prs)-1) extra := time.Duration(rand.Int63n(interval) - interval/2) @@ -104,13 +104,12 @@ func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Dura return len(prs), nil } -func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr PolicyRecord) { - log := xlog.WithContext(ctx) +func refreshDomain(ctx context.Context, log mlog.Log, db *bstore.DB, resolver dns.Resolver, pr PolicyRecord) { defer func() { x := recover() if x != nil { // Should not happen, but make sure errors don't take down the application. - log.Error("refresh1", mlog.Field("panic", x)) + log.Error("refresh1", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Mtastsdb) } @@ -121,11 +120,11 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr d, err := dns.ParseDomain(pr.Domain) if err != nil { - log.Errorx("refreshing mta-sts policy: parsing policy domain", err, mlog.Field("domain", d)) + log.Errorx("refreshing mta-sts policy: parsing policy domain", err, slog.Any("domain", d)) return } - log.Debug("refreshing mta-sts policy for domain", mlog.Field("domain", d)) - record, _, err := mtasts.LookupRecord(ctx, resolver, d) + log.Debug("refreshing mta-sts policy for domain", slog.Any("domain", d)) + record, _, err := mtasts.LookupRecord(ctx, log.Logger, resolver, d) if err == nil && record.ID == pr.RecordID { qup := bstore.QueryDB[PolicyRecord](ctx, db) qup.FilterNonzero(PolicyRecord{Domain: pr.Domain, LastUpdate: pr.LastUpdate}) @@ -137,7 +136,7 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr if n, err := qup.UpdateNonzero(update); err != nil { log.Errorx("updating refreshed, unmodified policy in database", err) } else if n != 1 { - log.Info("expected to update 1 policy after refresh", mlog.Field("count", n)) + log.Info("expected to update 1 policy after refresh", slog.Int("count", n)) } return } @@ -152,14 +151,14 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr // ../rfc/8461:587 return } else if err != nil { - log.Errorx("looking up mta-sts record for domain", err, mlog.Field("domain", d)) + log.Errorx("looking up mta-sts record for domain", err, slog.Any("domain", d)) // Try to fetch new policy. It could be just DNS that is down. We don't want to let our policy expire. } - p, _, err := mtasts.FetchPolicy(ctx, d) + p, _, err := mtasts.FetchPolicy(ctx, log.Logger, d) if err != nil { if !errors.Is(err, mtasts.ErrNoPolicy) || pr.Mode != mtasts.ModeNone { - log.Errorx("refreshing mtasts policy for domain", err, mlog.Field("domain", d)) + log.Errorx("refreshing mtasts policy for domain", err, slog.Any("domain", d)) } return } @@ -178,6 +177,6 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr if n, err := qup.UpdateFields(update); err != nil { log.Errorx("updating refreshed, modified policy in database", err) } else if n != 1 { - log.Info("updating refreshed, did not update 1 policy", mlog.Field("count", n)) + log.Info("updating refreshed, did not update 1 policy", slog.Int("count", n)) } } diff --git a/mtastsdb/refresh_test.go b/mtastsdb/refresh_test.go index 376c5e8..c9f08e6 100644 --- a/mtastsdb/refresh_test.go +++ b/mtastsdb/refresh_test.go @@ -22,6 +22,7 @@ import ( "github.com/mjl-/bstore" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mtasts" ) @@ -135,7 +136,8 @@ func TestRefresh(t *testing.T) { t.Fatalf("bad sleep duration %v", d) } } - if n, err := refresh1(ctxbg, resolver, sleep); err != nil || n != 3 { + log := mlog.New("mtastsdb", nil) + if n, err := refresh1(ctxbg, log, resolver, sleep); err != nil || n != 3 { t.Fatalf("refresh1: err %s, n %d, expected no error, 3", err, n) } if slept != 2 { diff --git a/publicsuffix/list.go b/publicsuffix/list.go index 850bd5b..90d046f 100644 --- a/publicsuffix/list.go +++ b/publicsuffix/list.go @@ -18,14 +18,13 @@ import ( _ "embed" + "golang.org/x/exp/slog" "golang.org/x/net/idna" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("publicsuffix") - // todo: automatically fetch new lists periodically? compare it with the old one. refuse it if it changed too much, especially if it contains far fewer entries than before. // Labels map from utf8 labels to labels for subdomains. @@ -43,16 +42,19 @@ var publicsuffixList List var publicsuffixData []byte func init() { - l, err := ParseList(bytes.NewReader(publicsuffixData)) + log := mlog.New("publicsuffix", nil) + l, err := ParseList(log.Logger, bytes.NewReader(publicsuffixData)) if err != nil { - xlog.Fatalx("parsing public suffix list", err) + log.Fatalx("parsing public suffix list", err) } publicsuffixList = l } // ParseList parses a public suffix list. // Only the "ICANN DOMAINS" are used. -func ParseList(r io.Reader) (List, error) { +func ParseList(elog *slog.Logger, r io.Reader) (List, error) { + log := mlog.New("publicsuffix", elog) + list := List{labels{}, labels{}} br := bufio.NewReader(r) @@ -79,7 +81,7 @@ func ParseList(r io.Reader) (List, error) { l = list.excludes t = strings.Split(line, ".") if len(t) == 1 { - xlog.Print("exclude rule with single label, skipping", mlog.Field("line", oline)) + log.Print("exclude rule with single label, skipping", slog.String("line", oline)) continue } } else { @@ -88,19 +90,19 @@ func ParseList(r io.Reader) (List, error) { for i := len(t) - 1; i >= 0; i-- { w := t[i] if w == "" { - xlog.Print("empty label in rule, skipping", mlog.Field("line", oline)) + log.Print("empty label in rule, skipping", slog.String("line", oline)) break } if w != "" && w != "*" { w, err = idna.Lookup.ToUnicode(w) if err != nil { - xlog.Printx("invalid label, skipping", err, mlog.Field("line", oline)) + log.Printx("invalid label, skipping", err, slog.String("line", oline)) } } m, ok := l[w] if ok { if _, dup := m[""]; i == 0 && dup { - xlog.Print("duplicate rule", mlog.Field("line", oline)) + log.Print("duplicate rule", slog.String("line", oline)) } l = m } else { @@ -123,16 +125,16 @@ func ParseList(r io.Reader) (List, error) { // Lookup calls Lookup on the builtin public suffix list, from // https://publicsuffix.org/list/. -func Lookup(ctx context.Context, domain dns.Domain) (orgDomain dns.Domain) { - return publicsuffixList.Lookup(ctx, domain) +func Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) { + return publicsuffixList.Lookup(ctx, elog, domain) } // Lookup returns the organizational domain. If domain is an organizational // domain, or higher-level, the same domain is returned. -func (l List) Lookup(ctx context.Context, domain dns.Domain) (orgDomain dns.Domain) { - log := xlog.WithContext(ctx) +func (l List) Lookup(ctx context.Context, elog *slog.Logger, domain dns.Domain) (orgDomain dns.Domain) { + log := mlog.New("publicsuffix", elog) defer func() { - log.Debug("publicsuffix lookup result", mlog.Field("reqdom", domain), mlog.Field("orgdom", orgDomain)) + log.Debug("publicsuffix lookup result", slog.Any("reqdom", domain), slog.Any("orgdom", orgDomain)) }() t := strings.Split(domain.Name(), ".") diff --git a/publicsuffix/list_test.go b/publicsuffix/list_test.go index fa5db72..538d9c6 100644 --- a/publicsuffix/list_test.go +++ b/publicsuffix/list_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" ) func TestList(t *testing.T) { @@ -27,7 +28,10 @@ bücher.example.com ignored.example.com ` - l, err := ParseList(strings.NewReader(data)) + + log := mlog.New("publicsuffix", nil) + + l, err := ParseList(log.Logger, strings.NewReader(data)) if err != nil { t.Fatalf("parsing list: %s", err) } @@ -44,7 +48,7 @@ ignored.example.com t.Fatalf("idna to unicode org domain %q: %s", orgDomain, err) } - r := l.Lookup(context.Background(), d) + r := l.Lookup(context.Background(), log.Logger, d) if r != od { t.Fatalf("got %q, expected %q, for domain %q", r, orgDomain, domain) } @@ -70,7 +74,7 @@ ignored.example.com test("bar.foo.xn--bcher-kva.example.com", "foo.bücher.example.com") test("x.ignored.example.com", "example.com") - l, err = ParseList(bytes.NewReader(publicsuffixData)) + l, err = ParseList(log.Logger, bytes.NewReader(publicsuffixData)) if err != nil { t.Fatalf("parsing public suffix list: %s", err) } diff --git a/queue/direct.go b/queue/direct.go index e0fe515..8d04d69 100644 --- a/queue/direct.go +++ b/queue/direct.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -88,7 +90,7 @@ var ( ) // todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time? -func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { +func fail(qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { // todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503 // todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls? // todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379 @@ -106,18 +108,18 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT qup := bstore.QueryDB[Msg](context.Background(), DB) qup.FilterID(m.ID) if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil { - qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg)) + qlog.Errorx("storing delivery error", err, slog.String("deliveryerror", errmsg)) } if m.Attempts == 5 { // We've attempted deliveries at these intervals: 0, 7.5m, 15m, 30m, 1h, 2u. // Let sender know delivery is delayed. - qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), mlog.Field("backoff", backoff)) + qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), slog.Duration("backoff", backoff)) retryUntil := m.LastAttempt.Add((4 + 8 + 16) * time.Hour) deliverDSNDelay(qlog, m, remoteMTA, secodeOpt, errmsg, retryUntil) } else { - qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), mlog.Field("backoff", backoff), mlog.Field("nextattempt", m.NextAttempt)) + qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), slog.Duration("backoff", backoff), slog.Time("nextattempt", m.NextAttempt)) } } @@ -129,7 +131,7 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT // domain (MTA-STS), its policy type can be empty, in which case there is no // information (e.g. internal failure). hostResults are per-host details (DANE, one // per MX target). -func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) (recipientDomainResult tlsrpt.Result, hostResults []tlsrpt.Result) { +func deliverDirect(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, m Msg, backoff time.Duration) (recipientDomainResult tlsrpt.Result, hostResults []tlsrpt.Result) { // High-level approach: // - Resolve domain to deliver to (CNAME), and determine hosts to try to deliver to (MX) // - Get MTA-STS policy for domain (optional). If present, only deliver to its @@ -151,8 +153,8 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp // possibly a chain. If there are no MX records, it can be an IP or the host // directly. origNextHop := m.RecipientDomain.Domain - ctx := context.WithValue(mox.Context, mlog.CidKey, cid) - haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog, resolver, m.RecipientDomain) + ctx := mox.Shutdown + haveMX, origNextHopAuthentic, expandedNextHopAuthentic, expandedNextHop, hosts, permanent, err := smtpclient.GatherDestinations(ctx, qlog.Logger, resolver, m.RecipientDomain) if err != nil { // If this is a DNSSEC authentication error, we'll collect it for TLS reporting. // Hopefully it's a temporary misconfiguration that is solve before we try to send @@ -179,14 +181,13 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp // would only take a single CNAME DNS response to direct us to an unrelated domain. var policy *mtasts.Policy // Policy can have mode enforce, testing and none. if !origNextHop.IsZero() { - cidctx := context.WithValue(mox.Shutdown, mlog.CidKey, cid) - policy, recipientDomainResult, _, err = mtastsdb.Get(cidctx, resolver, origNextHop) + policy, recipientDomainResult, _, err = mtastsdb.Get(ctx, qlog.Logger, resolver, origNextHop) if err != nil { if tlsRequiredNo { - qlog.Infox("mtasts lookup temporary error, continuing due to tls-required-no message header", err, mlog.Field("domain", origNextHop)) + qlog.Infox("mtasts lookup temporary error, continuing due to tls-required-no message header", err, slog.Any("domain", origNextHop)) metricTLSRequiredNoIgnored.WithLabelValues("mtastspolicy").Inc() } else { - qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, mlog.Field("domain", origNextHop)) + qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, slog.Any("domain", origNextHop)) recipientDomainResult.Summary.TotalFailureSessionCount++ fail(qlog, m, backoff, false, dsn.NameIP{}, "", err.Error()) return @@ -226,22 +227,21 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp } if policy.Mode == mtasts.ModeEnforce { if tlsRequiredNo { - qlog.Info("mx host does not match mta-sts policy in mode enforce, ignoring due to tls-required-no message header", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts)) + qlog.Info("mx host does not match mta-sts policy in mode enforce, ignoring due to tls-required-no message header", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts)) metricTLSRequiredNoIgnored.WithLabelValues("mtastsmx").Inc() } else { errmsg = fmt.Sprintf("mx host %s does not match enforced mta-sts policy with hosts %s", h.Domain, strings.Join(policyHosts, ",")) - qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts)) + qlog.Error("mx host does not match mta-sts policy in mode enforce, skipping", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts)) recipientDomainResult.Summary.TotalFailureSessionCount++ continue } } else { - qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", mlog.Field("host", h.Domain), mlog.Field("policyhosts", policyHosts)) + qlog.Error("mx host does not match mta-sts policy, but it is not enforced, continuing", slog.Any("host", h.Domain), slog.Any("policyhosts", policyHosts)) } } - qlog.Info("delivering to remote", mlog.Field("remote", h), mlog.Field("queuecid", cid)) - cid := mox.Cid() - nqlog := qlog.WithCid(cid) + qlog.Info("delivering to remote", slog.Any("remote", h)) + nqlog := qlog.WithCid(mox.Cid()) var remoteIP net.IP enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce @@ -270,7 +270,7 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp var badTLS, ok bool var hostResult tlsrpt.Result - permanent, tlsDANE, badTLS, secodeOpt, remoteIP, errmsg, hostResult, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode, tlsPKIX, &recipientDomainResult) + permanent, tlsDANE, badTLS, secodeOpt, remoteIP, errmsg, hostResult, ok = deliverHost(nqlog, resolver, dialer, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode, tlsPKIX, &recipientDomainResult) var zerotype tlsrpt.PolicyType if hostResult.Policy.Type != zerotype { @@ -293,8 +293,8 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp } // todo future: add a configuration option to not fall back? - nqlog.Info("connecting again for delivery attempt without tls", mlog.Field("enforcemtasts", enforceMTASTS), mlog.Field("tlsdane", tlsDANE), mlog.Field("requiretls", m.RequireTLS)) - permanent, _, _, secodeOpt, remoteIP, errmsg, _, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, smtpclient.TLSSkip, false, &tlsrpt.Result{}) + nqlog.Info("connecting again for delivery attempt without tls", slog.Bool("enforcemtasts", enforceMTASTS), slog.Bool("tlsdane", tlsDANE), slog.Any("requiretls", m.RequireTLS)) + permanent, _, _, secodeOpt, remoteIP, errmsg, _, ok = deliverHost(nqlog, resolver, dialer, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, smtpclient.TLSSkip, false, &tlsrpt.Result{}) } if ok { @@ -350,7 +350,7 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp // The returned hostResult holds TLSRPT reporting results for the connection // attempt. Its policy type can be the zero value, indicating there was no finding // (e.g. internal error). -func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, cid int64, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode, tlsPKIX bool, recipientDomainResult *tlsrpt.Result) (permanent, tlsDANE, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, hostResult tlsrpt.Result, ok bool) { +func deliverHost(log mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, ourHostname dns.Domain, transportName string, host dns.IPDomain, enforceMTASTS, haveMX, origNextHopAuthentic bool, origNextHop dns.Domain, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, m *Msg, tlsMode smtpclient.TLSMode, tlsPKIX bool, recipientDomainResult *tlsrpt.Result) (permanent, tlsDANE, badTLS bool, secodeOpt string, remoteIP net.IP, errmsg string, hostResult tlsrpt.Result, ok bool) { // About attempting delivery to multiple addresses of a host: ../rfc/5321:3898 tlsRequiredNo := m.RequireTLS != nil && !*m.RequireTLS @@ -367,18 +367,18 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, } metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, mode, deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second)) log.Debug("queue deliverhost result", - mlog.Field("host", host), - mlog.Field("attempt", m.Attempts), - mlog.Field("tlsmode", tlsMode), - mlog.Field("tlspkix", tlsPKIX), - mlog.Field("tlsdane", tlsDANE), - mlog.Field("tlsrequiredno", tlsRequiredNo), - mlog.Field("permanent", permanent), - mlog.Field("badtls", badTLS), - mlog.Field("secodeopt", secodeOpt), - mlog.Field("errmsg", errmsg), - mlog.Field("ok", ok), - mlog.Field("duration", time.Since(start))) + slog.Any("host", host), + slog.Int("attempt", m.Attempts), + slog.Any("tlsmode", tlsMode), + slog.Bool("tlspkix", tlsPKIX), + slog.Bool("tlsdane", tlsDANE), + slog.Bool("tlsrequiredno", tlsRequiredNo), + slog.Bool("permanent", permanent), + slog.Bool("badtls", badTLS), + slog.String("secodeopt", secodeOpt), + slog.String("errmsg", errmsg), + slog.Bool("ok", ok), + slog.Duration("duration", time.Since(start))) }() // Open message to deliver. @@ -392,8 +392,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, log.Check(err, "closing message after delivery attempt") }() - cidctx := context.WithValue(mox.Context, mlog.CidKey, cid) - ctx, cancel := context.WithTimeout(cidctx, 30*time.Second) + ctx, cancel := context.WithTimeout(mox.Shutdown, 30*time.Second) defer cancel() // We must lookup the IPs for the host name before checking DANE TLSA records. And @@ -415,10 +414,10 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, } metricDestinations.Inc() - authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log, resolver, host, m.DialedIPs) + authentic, expandedAuthentic, expandedHost, ips, dualstack, err := smtpclient.GatherIPs(ctx, log.Logger, resolver, host, m.DialedIPs) destAuthentic := err == nil && authentic && origNextHopAuthentic && (!haveMX || expandedNextHopAuthentic) && host.IsDomain() if !destAuthentic { - log.Debugx("not attempting verification with dane", err, mlog.Field("authentic", authentic), mlog.Field("expandedauthentic", expandedAuthentic)) + log.Debugx("not attempting verification with dane", err, slog.Bool("authentic", authentic), slog.Bool("expandedauthentic", expandedAuthentic)) // Track a DNSSEC error if found. var errCode adns.ErrorCode @@ -447,7 +446,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, // Look for TLSA records in either the expandedHost, or otherwise the original // host. ../rfc/7672:912 var tlsaBaseDomain dns.Domain - tlsDANE, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost) + tlsDANE, daneRecords, tlsaBaseDomain, err = smtpclient.GatherTLSA(ctx, log.Logger, resolver, host.Domain, expandedNextHopAuthentic && expandedAuthentic, expandedHost) if tlsDANE { metricDestinationDANERequired.Inc() } @@ -475,7 +474,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, }, } } else { - log.Debug("delivery with required starttls with dane verification", mlog.Field("allowedtlshostnames", tlsHostnames)) + log.Debug("delivery with required starttls with dane verification", slog.Any("allowedtlshostnames", tlsHostnames)) } // Based on CNAMEs followed and DNSSEC-secure status, we must allow up to 4 host // names. @@ -525,7 +524,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, if m.DialedIPs == nil { m.DialedIPs = map[string][]net.IP{} } - conn, remoteIP, err = smtpclient.Dial(ctx, log, dialer, host, ips, 25, m.DialedIPs) + conn, remoteIP, err = smtpclient.Dial(ctx, log.Logger, dialer, host, ips, 25, m.DialedIPs) } cancel() @@ -543,7 +542,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, } metricConnection.WithLabelValues(result).Inc() if err != nil { - log.Debugx("connecting to remote smtp", err, mlog.Field("host", host)) + log.Debugx("connecting to remote smtp", err, slog.Any("host", host)) return false, tlsDANE, false, "", remoteIP, fmt.Sprintf("dialing smtp server: %v", err), hostResult, false } @@ -554,8 +553,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, rcptTo := m.Recipient().XString(m.SMTPUTF8) // todo future: get closer to timeouts specified in rfc? ../rfc/5321:3610 - log = log.Fields(mlog.Field("remoteip", remoteIP)) - ctx, cancel = context.WithTimeout(cidctx, 30*time.Minute) + log = log.With(slog.Any("remoteip", remoteIP)) + ctx, cancel = context.WithTimeout(mox.Shutdown, 30*time.Minute) defer cancel() mox.Connections.Register(conn, "smtpclient", "queue") @@ -577,7 +576,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, RecipientDomainResult: recipientDomainResult, HostResult: &hostResult, } - sc, err := smtpclient.New(ctx, log, conn, tlsMode, tlsPKIX, ourHostname, firstHost, opts) + sc, err := smtpclient.New(ctx, log.Logger, conn, tlsMode, tlsPKIX, ourHostname, firstHost, opts) defer func() { if sc == nil { conn.Close() @@ -597,7 +596,7 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, STARTTLS: sc.TLSEnabled(), RequireTLS: sc.SupportsRequireTLS(), } - if err = updateRecipientDomainTLS(ctx, m.SenderAccount, rdt); err != nil { + if err = updateRecipientDomainTLS(ctx, log, m.SenderAccount, rdt); err != nil { err = fmt.Errorf("storing recipient domain tls status: %w", err) } } @@ -658,8 +657,8 @@ func deliverHost(log *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, } // Update (overwite) last known starttls/requiretls support for recipient domain. -func updateRecipientDomainTLS(ctx context.Context, senderAccount string, rdt store.RecipientDomainTLS) error { - acc, err := store.OpenAccount(senderAccount) +func updateRecipientDomainTLS(ctx context.Context, log mlog.Log, senderAccount string, rdt store.RecipientDomainTLS) error { + acc, err := store.OpenAccount(log, senderAccount) if err != nil { return fmt.Errorf("open account: %w", err) } diff --git a/queue/dsn.go b/queue/dsn.go index 73269c3..886b8cc 100644 --- a/queue/dsn.go +++ b/queue/dsn.go @@ -6,6 +6,8 @@ import ( "os" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -27,7 +29,7 @@ var ( ) ) -func deliverDSNFailure(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { +func deliverDSNFailure(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { const subject = "mail delivery failed" message := fmt.Sprintf(` Delivery has failed permanently for your email to: @@ -44,7 +46,7 @@ Error during the last delivery attempt: deliverDSN(log, m, remoteMTA, secodeOpt, errmsg, true, nil, subject, message) } -func deliverDSNDelay(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) { +func deliverDSNDelay(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) { // Should not happen, but doesn't hurt to prevent sending delayed delivery // notifications for DMARC reports. We don't want to waste postmaster attention. if m.IsDMARCReport { @@ -72,14 +74,14 @@ Error during the last delivery attempt: // users. So we are delivering to local users. ../rfc/5321:1466 // ../rfc/5321:1494 // ../rfc/7208:490 -func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) { +func deliverDSN(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) { kind := "delayed delivery" if permanent { kind = "failure" } qlog := func(text string, err error) { - log.Errorx("queue dsn: "+text+": sender will not be informed about dsn", err, mlog.Field("sender", m.Sender().XString(m.SMTPUTF8)), mlog.Field("kind", kind)) + log.Errorx("queue dsn: "+text+": sender will not be informed about dsn", err, slog.String("sender", m.Sender().XString(m.SMTPUTF8)), slog.String("kind", kind)) } msgf, err := os.Open(m.MessagePath()) @@ -156,9 +158,9 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st // senderAccount should already by postmaster, but doesn't hurt to ensure it. senderAccount = mox.Conf.Static.Postmaster.Account } - acc, err := store.OpenAccount(senderAccount) + acc, err := store.OpenAccount(log, senderAccount) if err != nil { - acc, err = store.OpenAccount(mox.Conf.Static.Postmaster.Account) + acc, err = store.OpenAccount(log, mox.Conf.Static.Postmaster.Account) if err != nil { qlog("looking up postmaster account after sender account was not found", err) return @@ -167,10 +169,10 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st } defer func() { err := acc.Close() - log.Check(err, "queue dsn: closing account", mlog.Field("sender", m.Sender().XString(m.SMTPUTF8)), mlog.Field("kind", kind)) + log.Check(err, "queue dsn: closing account", slog.String("sender", m.Sender().XString(m.SMTPUTF8)), slog.String("kind", kind)) }() - msgFile, err := store.CreateMessageTemp("queue-dsn") + msgFile, err := store.CreateMessageTemp(log, "queue-dsn") if err != nil { qlog("creating temporary message file", err) return diff --git a/queue/queue.go b/queue/queue.go index 2f85e3a..600842c 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "golang.org/x/exp/slog" "golang.org/x/net/proxy" "github.com/prometheus/client_golang/prometheus" @@ -36,8 +37,6 @@ import ( "github.com/mjl-/mox/tlsrptdb" ) -var xlog = mlog.New("queue") - var ( metricConnection = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -163,7 +162,9 @@ func Init() error { // Shutdown closes the queue database. The delivery process isn't stopped. For tests only. func Shutdown() { err := DB.Close() - xlog.Check(err, "closing queue db") + if err != nil { + mlog.New("queue", nil).Errorx("closing queue db", err) + } DB = nil } @@ -221,7 +222,7 @@ func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf // // Add sets derived fields like RecipientDomainStr, and fields related to queueing, // such as Queued, NextAttempt, LastAttempt, LastError. -func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error { +func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error { // todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759 if qm.ID != 0 { @@ -238,7 +239,7 @@ func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error { if qm.SenderAccount == "" { return fmt.Errorf("cannot queue with localserve without local account") } - acc, err := store.OpenAccount(qm.SenderAccount) + acc, err := store.OpenAccount(log, qm.SenderAccount) if err != nil { return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err) } @@ -279,14 +280,14 @@ func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error { defer func() { if dst != "" { err := os.Remove(dst) - log.Check(err, "removing destination message file for queue", mlog.Field("path", dst)) + log.Check(err, "removing destination message file for queue", slog.String("path", dst)) } }() dstDir := filepath.Dir(dst) os.MkdirAll(dstDir, 0770) if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil { return fmt.Errorf("linking/copying message to new file: %s", err) - } else if err := moxio.SyncDir(dstDir); err != nil { + } else if err := moxio.SyncDir(log, dstDir); err != nil { return fmt.Errorf("sync directory: %v", err) } @@ -359,7 +360,7 @@ func Kick(ctx context.Context, ID int64, toDomain, recipient string, transport * // Drop removes messages from the queue that match all nonzero parameters. // If all parameters are zero, all messages are removed. // Returns number of messages removed. -func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) { +func Drop(ctx context.Context, log mlog.Log, ID int64, toDomain string, recipient string) (int, error) { q := bstore.QueryDB[Msg](ctx, DB) if ID > 0 { q.FilterID(ID) @@ -381,7 +382,7 @@ func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int for _, m := range msgs { p := m.MessagePath() if err := os.Remove(p); err != nil { - xlog.WithContext(ctx).Errorx("removing queue message from file system", err, mlog.Field("queuemsgid", m.ID), mlog.Field("path", p)) + log.Errorx("removing queue message from file system", err, slog.Int64("queuemsgid", m.ID), slog.String("path", p)) } } return n, nil @@ -427,6 +428,8 @@ func Start(resolver dns.Resolver, done chan struct{}) error { return err } + log := mlog.New("queue", nil) + // High-level delivery strategy advice: ../rfc/5321:3685 go func() { // Map keys are either dns.Domain.Name()'s, or string-formatted IP addresses. @@ -449,14 +452,14 @@ func Start(resolver dns.Resolver, done chan struct{}) error { continue } - launchWork(resolver, busyDomains) - timer.Reset(nextWork(mox.Shutdown, busyDomains)) + launchWork(log, resolver, busyDomains) + timer.Reset(nextWork(mox.Shutdown, log, busyDomains)) } }() return nil } -func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration { +func nextWork(ctx context.Context, log mlog.Log, busyDomains map[string]struct{}) time.Duration { q := bstore.QueryDB[Msg](ctx, DB) if len(busyDomains) > 0 { var doms []any @@ -471,13 +474,13 @@ func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duratio if err == bstore.ErrAbsent { return 24 * time.Hour } else if err != nil { - xlog.Errorx("finding time for next delivery attempt", err) + log.Errorx("finding time for next delivery attempt", err) return 1 * time.Minute } return time.Until(qm.NextAttempt) } -func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int { +func launchWork(log mlog.Log, resolver dns.Resolver, busyDomains map[string]struct{}) int { q := bstore.QueryDB[Msg](mox.Shutdown, DB) q.FilterLessEqual("NextAttempt", time.Now()) q.SortAsc("NextAttempt") @@ -491,14 +494,14 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int { } msgs, err := q.List() if err != nil { - xlog.Errorx("querying for work in queue", err) + log.Errorx("querying for work in queue", err) mox.Sleep(mox.Shutdown, 1*time.Second) return -1 } for _, m := range msgs { busyDomains[formatIPDomain(m.RecipientDomain)] = struct{}{} - go deliver(resolver, m) + go deliver(log, resolver, m) } return len(msgs) } @@ -521,16 +524,15 @@ func queueDelete(ctx context.Context, msgID int64) error { // deliver attempts to deliver a message. // The queue is updated, either by removing a delivered or permanently failed // message, or updating the time for the next attempt. A DSN may be sent. -func deliver(resolver dns.Resolver, m Msg) { - cid := mox.Cid() - qlog := xlog.WithCid(cid).Fields(mlog.Field("from", m.Sender()), mlog.Field("recipient", m.Recipient()), mlog.Field("attempts", m.Attempts), mlog.Field("msgid", m.ID)) +func deliver(log mlog.Log, resolver dns.Resolver, m Msg) { + qlog := log.WithCid(mox.Cid()).With(slog.Any("from", m.Sender()), slog.Any("recipient", m.Recipient()), slog.Int("attempts", m.Attempts), slog.Int64("msgid", m.ID)) defer func() { deliveryResult <- formatIPDomain(m.RecipientDomain) x := recover() if x != nil { - qlog.Error("deliver panic", mlog.Field("panic", x)) + qlog.Error("deliver panic", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Queue) } @@ -578,8 +580,8 @@ func deliver(resolver dns.Resolver, m Msg) { } if transportName != "" { - qlog = qlog.Fields(mlog.Field("transport", transportName)) - qlog.Debug("delivering with transport", mlog.Field("transport", transportName)) + qlog = qlog.With(slog.String("transport", transportName)) + qlog.Debug("delivering with transport") } // We gather TLS connection successes and failures during delivery, and we store @@ -630,7 +632,7 @@ func deliver(resolver dns.Resolver, m Msg) { // Ensure we store policy domain in unicode in database. policyDomain, err := dns.ParseDomain(r.Policy.Domain) if err != nil { - qlog.Errorx("parsing policy domain for tls result", err, mlog.Field("policydomain", r.Policy.Domain)) + qlog.Errorx("parsing policy domain for tls result", err, slog.String("policydomain", r.Policy.Domain)) return } @@ -667,12 +669,12 @@ func deliver(resolver dns.Resolver, m Msg) { var dialer smtpclient.Dialer = &net.Dialer{} if transport.Submissions != nil { - deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465) + deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.Submissions, true, 465) } else if transport.Submission != nil { - deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.Submission, false, 587) + deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.Submission, false, 587) } else if transport.SMTP != nil { // todo future: perhaps also gather tlsrpt results for submissions. - deliverSubmit(cid, qlog, resolver, dialer, m, backoff, transportName, transport.SMTP, false, 25) + deliverSubmit(qlog, resolver, dialer, m, backoff, transportName, transport.SMTP, false, 25) } else { ourHostname := mox.Conf.Static.HostnameDomain if transport.Socks != nil { @@ -688,7 +690,7 @@ func deliver(resolver dns.Resolver, m Msg) { } ourHostname = transport.Socks.Hostname } - recipientDomainResult, hostResults = deliverDirect(cid, qlog, resolver, dialer, ourHostname, transportName, m, backoff) + recipientDomainResult, hostResults = deliverDirect(qlog, resolver, dialer, ourHostname, transportName, m, backoff) } } diff --git a/queue/queue_test.go b/queue/queue_test.go index 243de0a..d5ccc1f 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -23,6 +23,7 @@ import ( "github.com/mjl-/bstore" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtpclient" @@ -32,6 +33,7 @@ import ( ) var ctxbg = context.Background() +var pkglog = mlog.New("queue", nil) func tcheck(t *testing.T, err error, msg string) { if err != nil { @@ -61,12 +63,13 @@ func setup(t *testing.T) (*store.Account, func()) { os.RemoveAll("../testdata/queue/data/queue") } + log := mlog.New("queue", nil) mox.Context = ctxbg mox.ConfigStaticPath = filepath.FromSlash("../testdata/queue/mox.conf") mox.MustLoadConfig(true, false) - acc, err := store.OpenAccount("mjl") + acc, err := store.OpenAccount(log, "mjl") tcheck(t, err, "open account") - err = acc.SetPassword("testtest") + err = acc.SetPassword(log, "testtest") tcheck(t, err, "set password") switchStop := store.Switchboard() mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg) @@ -88,7 +91,7 @@ test email func prepareFile(t *testing.T) *os.File { t.Helper() - msgFile, err := store.CreateMessageTemp("queue") + msgFile, err := store.CreateMessageTemp(pkglog, "queue") tcheck(t, err, "create temp message for delivery to queue") _, err = msgFile.Write([]byte(testmsg)) tcheck(t, err, "write message file") @@ -115,11 +118,11 @@ func TestQueue(t *testing.T) { var qm Msg qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") msgs, err = List(ctxbg) @@ -131,7 +134,7 @@ func TestQueue(t *testing.T) { if msg.Attempts != 0 { t.Fatalf("msg attempts %d, expected 0", msg.Attempts) } - n, err := Drop(ctxbg, msgs[1].ID, "", "") + n, err := Drop(ctxbg, pkglog, msgs[1].ID, "", "") tcheck(t, err, "drop") if n != 1 { t.Fatalf("dropped %d, expected 1", n) @@ -140,15 +143,15 @@ func TestQueue(t *testing.T) { t.Fatalf("dropped message not removed from file system") } - next := nextWork(ctxbg, nil) + next := nextWork(ctxbg, pkglog, nil) if next > 0 { t.Fatalf("nextWork in %s, should be now", next) } busy := map[string]struct{}{"mox.example": {}} - if x := nextWork(ctxbg, busy); x != 24*time.Hour { + if x := nextWork(ctxbg, pkglog, busy); x != 24*time.Hour { t.Fatalf("nextWork in %s for busy domain, should be in 24 hours", x) } - if nn := launchWork(nil, busy); nn != 0 { + if nn := launchWork(pkglog, nil, busy); nn != 0 { t.Fatalf("launchWork launched %d deliveries, expected 0", nn) } @@ -171,7 +174,7 @@ func TestQueue(t *testing.T) { smtpclient.DialHook = nil }() - launchWork(resolver, map[string]struct{}{}) + launchWork(pkglog, resolver, map[string]struct{}{}) moxCert := fakeCert(t, "mail.mox.example", false) @@ -427,7 +430,7 @@ func TestQueue(t *testing.T) { <-deliveryResult // Deliver sends here. } - launchWork(resolver, map[string]struct{}{}) + launchWork(pkglog, resolver, map[string]struct{}{}) waitDeliver() return wasNetDialer } @@ -449,7 +452,7 @@ func TestQueue(t *testing.T) { // Add a message to be delivered with submit because of its route. topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}} qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") wasNetDialer = testDeliver(fakeSubmitServer) if !wasNetDialer { @@ -458,7 +461,7 @@ func TestQueue(t *testing.T) { // Add a message to be delivered with submit because of explicitly configured transport, that uses TLS. qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") transportSubmitTLS := "submittls" n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS) @@ -507,7 +510,7 @@ func TestQueue(t *testing.T) { // Add a message to be delivered with socks. qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") transportSocks := "socks" n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks) @@ -523,7 +526,7 @@ func TestQueue(t *testing.T) { // Add message to be delivered with opportunistic TLS verification. clearTLSResults(t) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -537,7 +540,7 @@ func TestQueue(t *testing.T) { // Test fallback to plain text with TLS handshake fails. clearTLSResults(t) qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -557,7 +560,7 @@ func TestQueue(t *testing.T) { }, } qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -578,7 +581,7 @@ func TestQueue(t *testing.T) { // Add message to be delivered with verified TLS and REQUIRETLS. yes := true qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &yes) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -595,7 +598,7 @@ func TestQueue(t *testing.T) { }, } qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -616,7 +619,7 @@ func TestQueue(t *testing.T) { }, } qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -638,7 +641,7 @@ func TestQueue(t *testing.T) { // Check that message is delivered with TLS-Required: No and non-matching DANE record. no := false qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &no) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -649,7 +652,7 @@ func TestQueue(t *testing.T) { // Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text. qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &no) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -660,7 +663,7 @@ func TestQueue(t *testing.T) { // Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers. qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &yes) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -675,7 +678,7 @@ func TestQueue(t *testing.T) { // Add message with requiretls that fails immediately due to no verification policy for recipient domain. qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &yes) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") @@ -690,7 +693,7 @@ func TestQueue(t *testing.T) { // Add another message that we'll fail to deliver entirely. qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") msgs, err = List(ctxbg) @@ -756,7 +759,7 @@ func TestQueue(t *testing.T) { resolver.AllAuthentic = false resolver.TLSA = nil } - deliver(resolver, msg) + deliver(pkglog, resolver, msg) err = DB.Get(ctxbg, &msg) tcheck(t, err, "get msg") if msg.Attempts != i { @@ -779,7 +782,7 @@ func TestQueue(t *testing.T) { // Trigger final failure. go func() { <-deliveryResult }() // Deliver sends here. - deliver(resolver, msg) + deliver(pkglog, resolver, msg) err = DB.Get(ctxbg, &msg) if err != bstore.ErrAbsent { t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err) @@ -881,7 +884,7 @@ func TestQueueStart(t *testing.T) { defer os.Remove(mf.Name()) defer mf.Close() qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) - err = Add(ctxbg, xlog, &qm, mf) + err = Add(ctxbg, pkglog, &qm, mf) tcheck(t, err, "add message to queue for delivery") checkDialed(true) diff --git a/queue/submit.go b/queue/submit.go index 66f53d0..d61d45e 100644 --- a/queue/submit.go +++ b/queue/submit.go @@ -10,6 +10,8 @@ import ( "os" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/config" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/dsn" @@ -25,7 +27,7 @@ import ( // deliver via another SMTP server, e.g. relaying to a smart host, possibly // with authentication (submission). -func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) { +func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer, m Msg, backoff time.Duration, transportName string, transport *config.TransportSMTP, dialTLS bool, defaultPort int) { // todo: configurable timeouts port := transport.Port @@ -52,7 +54,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp var success bool defer func() { metricDelivery.WithLabelValues(fmt.Sprintf("%d", m.Attempts), transportName, string(tlsMode), deliveryResult).Observe(float64(time.Since(start)) / float64(time.Second)) - qlog.Debug("queue deliversubmit result", mlog.Field("host", transport.DNSHost), mlog.Field("port", port), mlog.Field("attempt", m.Attempts), mlog.Field("permanent", permanent), mlog.Field("secodeopt", secodeOpt), mlog.Field("errmsg", errmsg), mlog.Field("ok", success), mlog.Field("duration", time.Since(start))) + qlog.Debug("queue deliversubmit result", slog.Any("host", transport.DNSHost), slog.Int("port", port), slog.Int("attempt", m.Attempts), slog.Bool("permanent", permanent), slog.String("secodeopt", secodeOpt), slog.String("errmsg", errmsg), slog.Bool("ok", success), slog.Duration("duration", time.Since(start))) }() // todo: SMTP-DANE should be used when relaying on port 25. @@ -74,13 +76,13 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp if m.DialedIPs == nil { m.DialedIPs = map[string][]net.IP{} } - _, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs) + _, _, _, ips, _, err := smtpclient.GatherIPs(dialctx, qlog.Logger, resolver, dns.IPDomain{Domain: transport.DNSHost}, m.DialedIPs) var conn net.Conn if err == nil { if m.DialedIPs == nil { m.DialedIPs = map[string][]net.IP{} } - conn, _, err = smtpclient.Dial(dialctx, qlog, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs) + conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs) } addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port)) var result string @@ -100,7 +102,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp err := conn.Close() qlog.Check(err, "closing connection") } - qlog.Errorx("dialing for submission", err, mlog.Field("remote", addr)) + qlog.Errorx("dialing for submission", err, slog.String("remote", addr)) errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err) fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) return @@ -122,7 +124,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp auth = append(auth, sasl.NewClientSCRAMSHA256(a.Username, a.Password)) default: // Should not happen. - qlog.Error("missing smtp authentication mechanisms implementation", mlog.Field("mechanism", mech)) + qlog.Error("missing smtp authentication mechanisms implementation", slog.String("mechanism", mech)) errmsg = fmt.Sprintf("transport %s: authentication mechanisms %q not implemented", transportName, mech) fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) return @@ -135,14 +137,14 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp Auth: auth, RootCAs: mox.Conf.Static.TLS.CertPool, } - client, err := smtpclient.New(clientctx, qlog, conn, tlsMode, tlsPKIX, mox.Conf.Static.HostnameDomain, transport.DNSHost, opts) + client, err := smtpclient.New(clientctx, qlog.Logger, conn, tlsMode, tlsPKIX, mox.Conf.Static.HostnameDomain, transport.DNSHost, opts) if err != nil { smtperr, ok := err.(smtpclient.Error) var remoteMTA dsn.NameIP if ok { remoteMTA.Name = transport.Host } - qlog.Errorx("establishing smtp session for submission", err, mlog.Field("remote", addr)) + qlog.Errorx("establishing smtp session for submission", err, slog.String("remote", addr)) errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err) secodeOpt = smtperr.Secode fail(qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg) @@ -168,7 +170,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp p := m.MessagePath() f, err := os.Open(p) if err != nil { - qlog.Errorx("opening message for delivery", err, mlog.Field("remote", addr), mlog.Field("path", p)) + qlog.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p)) errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err) fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) return @@ -209,7 +211,7 @@ func deliverSubmit(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp if ok { remoteMTA.Name = transport.Host } - qlog.Errorx("submitting email", err, mlog.Field("remote", addr)) + qlog.Errorx("submitting email", err, slog.String("remote", addr)) permanent = smtperr.Permanent secodeOpt = smtperr.Secode errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err) diff --git a/quickstart.go b/quickstart.go index ef64880..c916021 100644 --- a/quickstart.go +++ b/quickstart.go @@ -535,7 +535,7 @@ messages over SMTP. for _, zone := range zones { for _, ip := range hostIPs { dnsblctx, dnsblcancel := context.WithTimeout(resolveCtx, 5*time.Second) - status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(ip)) + status, expl, err := dnsbl.Lookup(dnsblctx, c.log.Logger, resolver, zone, net.ParseIP(ip)) dnsblcancel() if status == dnsbl.StatusPass { continue @@ -813,7 +813,7 @@ and check the admin page for the needed DNS records.`) // Verify config. loadTLSKeyCerts := !existingWebserver - mc, errs := mox.ParseConfig(context.Background(), filepath.FromSlash("config/mox.conf"), true, loadTLSKeyCerts, false) + mc, errs := mox.ParseConfig(context.Background(), c.log, filepath.FromSlash("config/mox.conf"), true, loadTLSKeyCerts, false) if len(errs) > 0 { if len(errs) > 1 { log.Printf("checking generated config, multiple errors:") @@ -834,14 +834,14 @@ and check the admin page for the needed DNS records.`) fatalf("cannot find domain in new config") } - acc, _, err := store.OpenEmail(args[0]) + acc, _, err := store.OpenEmail(c.log, args[0]) if err != nil { fatalf("open account: %s", err) } cleanupPaths = append(cleanupPaths, dataDir, filepath.Join(dataDir, "accounts"), filepath.Join(dataDir, "accounts", accountName), filepath.Join(dataDir, "accounts", accountName, "index.db")) password := pwgen() - if err := acc.SetPassword(password); err != nil { + if err := acc.SetPassword(c.log, password); err != nil { fatalf("setting password: %s", err) } if err := acc.Close(); err != nil { diff --git a/sendmail.go b/sendmail.go index 3adb5db..f27df75 100644 --- a/sendmail.go +++ b/sendmail.go @@ -17,7 +17,6 @@ import ( "github.com/mjl-/sconf" "github.com/mjl-/mox/dns" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/sasl" "github.com/mjl-/mox/smtp" @@ -300,7 +299,7 @@ binary should be setgid that group: Auth: auth, RootCAs: mox.Conf.Static.TLS.CertPool, } - client, err := smtpclient.New(ctx, mlog.New("sendmail"), conn, tlsMode, tlsPKIX, ourHostname, remoteHostname, opts) + client, err := smtpclient.New(ctx, c.log.Logger, conn, tlsMode, tlsPKIX, ourHostname, remoteHostname, opts) xsavecheckf(err, "open smtp session") err = client.Deliver(ctx, submitconf.From, recipient, int64(len(msg)), strings.NewReader(msg), true, false, submitconf.RequireTLS == RequireTLSYes) diff --git a/serve.go b/serve.go index 7f50f90..1457a5e 100644 --- a/serve.go +++ b/serve.go @@ -19,7 +19,7 @@ import ( "github.com/mjl-/mox/tlsrptsend" ) -func shutdown(log *mlog.Log) { +func shutdown(log mlog.Log) { // We indicate we are shutting down. Causes new connections and new SMTP commands // to be rejected. Should stop active connections pretty quickly. mox.ShutdownCancel() diff --git a/serve_unix.go b/serve_unix.go index 63fad09..8662781 100644 --- a/serve_unix.go +++ b/serve_unix.go @@ -18,6 +18,8 @@ import ( "syscall" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -32,12 +34,12 @@ import ( "github.com/mjl-/mox/updates" ) -func monitorDNSBL(log *mlog.Log) { +func monitorDNSBL(log mlog.Log) { defer func() { // On error, don't bring down the entire server. x := recover() if x != nil { - log.Error("monitordnsbl panic", mlog.Field("panic", x)) + log.Error("monitordnsbl panic", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Serve) } @@ -53,7 +55,7 @@ func monitorDNSBL(log *mlog.Log) { for _, zone := range l.SMTP.DNSBLs { d, err := dns.ParseDomain(zone) if err != nil { - log.Fatalx("parsing dnsbls zone", err, mlog.Field("zone", zone)) + log.Fatalx("parsing dnsbls zone", err, slog.Any("zone", zone)) } zones = append(zones, d) } @@ -86,9 +88,9 @@ func monitorDNSBL(log *mlog.Log) { } for _, zone := range zones { - status, expl, err := dnsbl.Lookup(mox.Context, resolver, zone, ip) + status, expl, err := dnsbl.Lookup(mox.Context, log.Logger, resolver, zone, ip) if err != nil { - log.Errorx("dnsbl monitor lookup", err, mlog.Field("ip", ip), mlog.Field("zone", zone), mlog.Field("expl", expl), mlog.Field("status", status)) + log.Errorx("dnsbl monitor lookup", err, slog.Any("ip", ip), slog.Any("zone", zone), slog.String("expl", expl), slog.Any("status", status)) } k := key{zone, ip.String()} @@ -145,7 +147,7 @@ Only implemented on unix systems, not Windows. checkACMEHosts := os.Getuid() != 0 - log := mlog.New("serve") + log := c.log if os.Getuid() == 0 { mox.MustLoadConfig(true, checkACMEHosts) @@ -159,7 +161,7 @@ Only implemented on unix systems, not Windows. domainsconf, err := filepath.Abs(mox.ConfigDynamicPath) log.Check(err, "finding absolute domains.conf path") - log.Print("starting as root, initializing network listeners", mlog.Field("version", moxvar.Version), mlog.Field("pid", os.Getpid()), mlog.Field("moxconf", moxconf), mlog.Field("domainsconf", domainsconf)) + log.Print("starting as root, initializing network listeners", slog.String("version", moxvar.Version), slog.Any("pid", os.Getpid()), slog.String("moxconf", moxconf), slog.String("domainsconf", domainsconf)) if os.Getenv("MOX_SOCKETS") != "" { log.Fatal("refusing to start as root with $MOX_SOCKETS set") } @@ -185,7 +187,7 @@ Only implemented on unix systems, not Windows. } else { mox.RestorePassedFiles() mox.MustLoadConfig(true, checkACMEHosts) - log.Print("starting as unprivileged user", mlog.Field("user", mox.Conf.Static.User), mlog.Field("uid", mox.Conf.Static.UID), mlog.Field("gid", mox.Conf.Static.GID), mlog.Field("pid", os.Getpid())) + log.Print("starting as unprivileged user", slog.String("user", mox.Conf.Static.User), slog.Any("uid", mox.Conf.Static.UID), slog.Any("gid", mox.Conf.Static.GID), slog.Any("pid", os.Getpid())) } syscall.Umask(syscall.Umask(007) | 007) @@ -200,12 +202,12 @@ Only implemented on unix systems, not Windows. log.Fatalx("reading random recvid data", err) } if err := os.WriteFile(recvidpath, recvidbuf, 0660); err != nil { - log.Fatalx("writing recvidpath", err, mlog.Field("path", recvidpath)) + log.Fatalx("writing recvidpath", err, slog.String("path", recvidpath)) } err := os.Chown(recvidpath, int(mox.Conf.Static.UID), 0) - log.Check(err, "chown receveidid.key", mlog.Field("path", recvidpath), mlog.Field("uid", mox.Conf.Static.UID), mlog.Field("gid", 0)) + log.Check(err, "chown receveidid.key", slog.String("path", recvidpath), slog.Any("uid", mox.Conf.Static.UID), slog.Any("gid", 0)) err = os.Chmod(recvidpath, 0640) - log.Check(err, "chmod receveidid.key to 0640", mlog.Field("path", recvidpath)) + log.Check(err, "chmod receveidid.key to 0640", slog.String("path", recvidpath)) } if err := mox.ReceivedIDInit(recvidbuf[:16], recvidbuf[16:]); err != nil { log.Fatalx("init receivedid", err) @@ -242,7 +244,7 @@ Only implemented on unix systems, not Windows. // mtime. But file won't exist initially. if !mtime.IsZero() && time.Since(mtime) < 24*time.Hour { d := 24*time.Hour - time.Since(mtime) - log.Debug("sleeping for next check for updates", mlog.Field("sleep", d)) + log.Debug("sleeping for next check for updates", slog.Duration("sleep", d)) time.Sleep(d) next = 0 } @@ -253,12 +255,12 @@ Only implemented on unix systems, not Windows. } } - log.Debug("checking for updates", mlog.Field("lastknown", lastknown)) + log.Debug("checking for updates", slog.Any("lastknown", lastknown)) updatesctx, updatescancel := context.WithTimeout(mox.Context, time.Minute) - latest, _, changelog, err := updates.Check(updatesctx, dns.StrictResolver{}, dns.Domain{ASCII: changelogDomain}, lastknown, changelogURL, changelogPubKey) + latest, _, changelog, err := updates.Check(updatesctx, log.Logger, dns.StrictResolver{Log: log.Logger}, dns.Domain{ASCII: changelogDomain}, lastknown, changelogURL, changelogPubKey) updatescancel() if err != nil { - log.Infox("checking for updates", err, mlog.Field("latest", latest)) + log.Infox("checking for updates", err, slog.Any("latest", latest)) return next } if !latest.After(lastknown) { @@ -266,7 +268,7 @@ Only implemented on unix systems, not Windows. return next } if len(changelog.Changes) == 0 { - log.Info("new version available, but changelog is empty, ignoring", mlog.Field("latest", latest)) + log.Info("new version available, but changelog is empty, ignoring", slog.Any("latest", latest)) return next } @@ -276,7 +278,7 @@ Only implemented on unix systems, not Windows. } cl += "----" - a, err := store.OpenAccount(mox.Conf.Static.Postmaster.Account) + a, err := store.OpenAccount(log, mox.Conf.Static.Postmaster.Account) if err != nil { log.Infox("open account for postmaster changelog delivery", err) return next @@ -285,7 +287,7 @@ Only implemented on unix systems, not Windows. err := a.Close() log.Check(err, "closing account") }() - f, err := store.CreateMessageTemp("changelog") + f, err := store.CreateMessageTemp(log, "changelog") if err != nil { log.Infox("making temporary message file for changelog delivery", err) return next @@ -306,7 +308,7 @@ Only implemented on unix systems, not Windows. log.Errorx("changelog delivery", err) return next } - log.Info("delivered changelog", mlog.Field("current", current), mlog.Field("lastknown", lastknown), mlog.Field("latest", latest)) + log.Info("delivered changelog", slog.Any("current", current), slog.Any("lastknown", lastknown), slog.Any("latest", latest)) if err := mox.StoreLastKnown(latest); err != nil { // This will be awkward, we'll keep notifying the postmaster once every 24h... log.Infox("updating last known version", err) @@ -353,13 +355,13 @@ Only implemented on unix systems, not Windows. now := time.Now() for _, e := range tmps { if fi, err := e.Info(); err != nil { - log.Errorx("stat tmp file", err, mlog.Field("filename", e.Name())) + log.Errorx("stat tmp file", err, slog.String("filename", e.Name())) } else if now.Sub(fi.ModTime()) > 7*24*time.Hour && !fi.IsDir() { p := filepath.Join(tmpdir, e.Name()) if err := os.Remove(p); err != nil { - log.Errorx("removing stale temporary file", err, mlog.Field("path", p)) + log.Errorx("removing stale temporary file", err, slog.String("path", p)) } else { - log.Info("removed stale temporary file", mlog.Field("path", p)) + log.Info("removed stale temporary file", slog.String("path", p)) } } } @@ -369,7 +371,7 @@ Only implemented on unix systems, not Windows. sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) sig := <-sigc - log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig)) + log.Print("shutting down, waiting max 3s for existing connections", slog.Any("signal", sig)) shutdown(log) if num, ok := sig.(syscall.Signal); ok { os.Exit(int(num)) @@ -383,7 +385,7 @@ Only implemented on unix systems, not Windows. // We require being able to stat the basic non-optional paths. Then we'll try to // fix up permissions. If an error occurs when fixing permissions, we log and // continue (could not be an actual problem). -func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid uint32) (rerr error) { +func fixperms(log mlog.Log, workdir, configdir, datadir string, moxuid, moxgid uint32) (rerr error) { type fserr struct{ Err error } defer func() { x := recover() @@ -483,33 +485,33 @@ func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid for _, ch := range changes { if ch.uid != nil { err := os.Chown(ch.path, int(*ch.uid), int(*ch.gid)) - log.Printx("chown, fixing uid/gid", err, mlog.Field("path", ch.path), mlog.Field("olduid", ch.olduid), mlog.Field("oldgid", ch.oldgid), mlog.Field("newuid", *ch.uid), mlog.Field("newgid", *ch.gid)) + log.Printx("chown, fixing uid/gid", err, slog.String("path", ch.path), slog.Any("olduid", ch.olduid), slog.Any("oldgid", ch.oldgid), slog.Any("newuid", *ch.uid), slog.Any("newgid", *ch.gid)) } if ch.mode != nil { err := os.Chmod(ch.path, *ch.mode) - log.Printx("chmod, fixing permissions", err, mlog.Field("path", ch.path), mlog.Field("oldmode", fmt.Sprintf("%03o", ch.oldmode)), mlog.Field("newmode", fmt.Sprintf("%03o", *ch.mode))) + log.Printx("chmod, fixing permissions", err, slog.String("path", ch.path), slog.Any("oldmode", fmt.Sprintf("%03o", ch.oldmode)), slog.Any("newmode", fmt.Sprintf("%03o", *ch.mode))) } } walkchange := func(dir string) { err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil { - log.Printx("walk error, continuing", err, mlog.Field("path", path)) + log.Printx("walk error, continuing", err, slog.String("path", path)) return nil } fi, err := d.Info() if err != nil { - log.Printx("stat during walk, continuing", err, mlog.Field("path", path)) + log.Printx("stat during walk, continuing", err, slog.String("path", path)) return nil } st, ok := fi.Sys().(*syscall.Stat_t) if !ok { - log.Printx("syscall stat during walk, continuing", err, mlog.Field("path", path)) + log.Printx("syscall stat during walk, continuing", err, slog.String("path", path)) return nil } if st.Uid != moxuid || st.Gid != root { err := os.Chown(path, int(moxuid), root) - log.Printx("walk chown, fixing uid/gid", err, mlog.Field("path", path), mlog.Field("olduid", st.Uid), mlog.Field("oldgid", st.Gid), mlog.Field("newuid", moxuid), mlog.Field("newgid", root)) + log.Printx("walk chown, fixing uid/gid", err, slog.String("path", path), slog.Any("olduid", st.Uid), slog.Any("oldgid", st.Gid), slog.Any("newuid", moxuid), slog.Any("newgid", root)) } omode := fi.Mode() & (fs.ModeSetgid | 0777) var nmode fs.FileMode @@ -520,21 +522,21 @@ func fixperms(log *mlog.Log, workdir, configdir, datadir string, moxuid, moxgid } if omode != nmode { err := os.Chmod(path, nmode) - log.Printx("walk chmod, fixing permissions", err, mlog.Field("path", path), mlog.Field("oldmode", fmt.Sprintf("%03o", omode)), mlog.Field("newmode", fmt.Sprintf("%03o", nmode))) + log.Printx("walk chmod, fixing permissions", err, slog.String("path", path), slog.Any("oldmode", fmt.Sprintf("%03o", omode)), slog.Any("newmode", fmt.Sprintf("%03o", nmode))) } return nil }) - log.Check(err, "walking dir to fix permissions", mlog.Field("dir", dir)) + log.Check(err, "walking dir to fix permissions", slog.String("dir", dir)) } // If config or data dir needed fixing, also set uid/gid and mode and files/dirs // inside, recursively. We don't always recurse, data probably contains many files. if fixconfig { - log.Print("fixing permissions in config dir", mlog.Field("configdir", configdir)) + log.Print("fixing permissions in config dir", slog.String("configdir", configdir)) walkchange(configdir) } if fixdata { - log.Print("fixing permissions in data dir", mlog.Field("configdir", configdir)) + log.Print("fixing permissions in data dir", slog.String("configdir", configdir)) walkchange(datadir) } return nil diff --git a/smtpclient/client.go b/smtpclient/client.go index 7d97bdd..93d925d 100644 --- a/smtpclient/client.go +++ b/smtpclient/client.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -118,7 +120,7 @@ type Client struct { w *bufio.Writer tr *moxio.TraceReader // Kept for changing trace levels between cmd/auth/data. tw *moxio.TraceWriter - log *mlog.Log + log mlog.Log lastlog time.Time // For adding delta timestamps between log lines. cmds []string // Last or active command, for generating errors and metrics. cmdStart time.Time // Start of command. @@ -237,7 +239,7 @@ type Opts struct { // with opportunistic TLS without PKIX verification by default. Recipient domains // can opt-in to PKIX verification by publishing an MTA-STS policy, or opt-in to // DANE verification by publishing DNSSEC-protected TLSA records in DNS. -func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tlsVerifyPKIX bool, ehloHostname, remoteHostname dns.Domain, opts Opts) (*Client, error) { +func New(ctx context.Context, elog *slog.Logger, conn net.Conn, tlsMode TLSMode, tlsVerifyPKIX bool, ehloHostname, remoteHostname dns.Domain, opts Opts) (*Client, error) { ensureResult := func(r *tlsrpt.Result) *tlsrpt.Result { if r == nil { return &tlsrpt.Result{} @@ -259,10 +261,10 @@ func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tls recipientDomainResult: ensureResult(opts.RecipientDomainResult), hostResult: ensureResult(opts.HostResult), } - c.log = log.Fields(mlog.Field("smtpclient", "")).MoreFields(func() []mlog.Pair { + c.log = mlog.New("smtpclient", elog).WithFunc(func() []slog.Attr { now := time.Now() - l := []mlog.Pair{ - mlog.Field("delta", now.Sub(c.lastlog)), + l := []slog.Attr{ + slog.Duration("delta", now.Sub(c.lastlog)), } c.lastlog = now return l @@ -280,7 +282,7 @@ func New(ctx context.Context, log *mlog.Log, conn net.Conn, tlsMode TLSMode, tls c.tlsResultAdd(1, 0, nil) c.conn = tlsconn tlsversion, ciphersuite := mox.TLSInfo(tlsconn) - c.log.Debug("tls client handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite), mlog.Field("servername", remoteHostname)) + c.log.Debug("tls client handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite), slog.Any("servername", remoteHostname)) c.tls = true } else { c.conn = conn @@ -329,8 +331,8 @@ func (c *Client) tlsConfig() *tls.Config { // DANE verification. // daneRecords can be non-nil and empty, that's intended. if c.daneRecords != nil { - verified, record, err := dane.Verify(c.log, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames) - c.log.Debugx("dane verification", err, mlog.Field("verified", verified), mlog.Field("record", record)) + verified, record, err := dane.Verify(c.log.Logger, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames) + c.log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record)) if verified { if c.daneVerifiedRecord != nil { *c.daneVerifiedRecord = record @@ -426,7 +428,7 @@ func (c *Client) xerrorf(permanent bool, code int, secode, lastLine, format stri type timeoutWriter struct { conn net.Conn timeout time.Duration - log *mlog.Log + log mlog.Log } func (w timeoutWriter) Write(buf []byte) (int, error) { @@ -445,7 +447,7 @@ func (c *Client) readline() (string, error) { c.log.Errorx("setting read deadline", err) } - line, err := bufs.Readline(c.r) + line, err := bufs.Readline(c.log, c.r) if err != nil { // See if this is a TLS alert from remote, and one other than 0 (which notifies // that the connection is being closed. If so, we register a TLS connection @@ -463,7 +465,7 @@ func (c *Client) readline() (string, error) { return line, nil } -func (c *Client) xtrace(level mlog.Level) func() { +func (c *Client) xtrace(level slog.Level) func() { c.xflush() c.tr.SetTrace(level) c.tw.SetTrace(level) @@ -543,7 +545,7 @@ func (c *Client) readecode(ecodes bool) (code int, secode, lastLine string, text } } metricCommands.WithLabelValues(cmd, fmt.Sprintf("%d", co), sec).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) - c.log.Debug("smtpclient command result", mlog.Field("cmd", cmd), mlog.Field("code", co), mlog.Field("secode", sec), mlog.Field("duration", time.Since(c.cmdStart))) + c.log.Debug("smtpclient command result", slog.String("cmd", cmd), slog.Int("code", co), slog.String("secode", sec), slog.Duration("duration", time.Since(c.cmdStart))) } return co, sec, line, texts, nil } @@ -726,7 +728,7 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do // Attempt TLS if remote understands STARTTLS and we aren't doing immediate TLS or if caller requires it. if c.extStartTLS && tlsMode == TLSOpportunistic || tlsMode == TLSRequiredStartTLS { - c.log.Debug("starting tls client", mlog.Field("tlsmode", tlsMode), mlog.Field("servername", c.remoteHostname)) + c.log.Debug("starting tls client", slog.Any("tlsmode", tlsMode), slog.Any("servername", c.remoteHostname)) c.cmds[0] = "starttls" c.cmdStart = time.Now() c.xwritelinef("STARTTLS") @@ -772,14 +774,14 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do tlsversion, ciphersuite := mox.TLSInfo(nconn) c.log.Debug("starttls client handshake done", - mlog.Field("tlsmode", tlsMode), - mlog.Field("verifypkix", c.tlsVerifyPKIX), - mlog.Field("verifydane", c.daneRecords != nil), - mlog.Field("ignoretlsverifyerrors", c.ignoreTLSVerifyErrors), - mlog.Field("tls", tlsversion), - mlog.Field("ciphersuite", ciphersuite), - mlog.Field("servername", c.remoteHostname), - mlog.Field("danerecord", c.daneVerifiedRecord)) + slog.Any("tlsmode", tlsMode), + slog.Bool("verifypkix", c.tlsVerifyPKIX), + slog.Bool("verifydane", c.daneRecords != nil), + slog.Bool("ignoretlsverifyerrors", c.ignoreTLSVerifyErrors), + slog.String("tls", tlsversion), + slog.String("ciphersuite", ciphersuite), + slog.Any("servername", c.remoteHostname), + slog.Any("danerecord", c.daneVerifiedRecord)) c.tls = true // Track successful TLS connection. ../rfc/8460:515 c.tlsResultAdd(1, 0, nil) @@ -1171,7 +1173,7 @@ func (c *Client) Close() (rerr error) { c.xwriteline("QUIT") if err := c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { c.log.Infox("setting read deadline for reading quit response", err) - } else if _, err := bufs.Readline(c.r); err != nil { + } else if _, err := bufs.Readline(c.log, c.r); err != nil { rerr = fmt.Errorf("reading response to quit command: %v", err) c.log.Debugx("reading quit response", err) } diff --git a/smtpclient/client_test.go b/smtpclient/client_test.go index af28924..0728e48 100644 --- a/smtpclient/client_test.go +++ b/smtpclient/client_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/sasl" @@ -33,9 +35,9 @@ var localhost = dns.Domain{ASCII: "localhost"} func TestClient(t *testing.T) { ctx := context.Background() - log := mlog.New("smtpclient") + log := mlog.New("smtpclient", nil) - mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelTrace}) + mlog.SetConfig(map[string]slog.Level{"": mlog.LevelTrace}) type options struct { pipelining bool @@ -281,7 +283,7 @@ func TestClient(t *testing.T) { result <- err panic("stop") } - c, err := New(ctx, log, clientConn, opts.tlsMode, opts.tlsPKIX, localhost, opts.tlsHostname, Opts{Auth: auths, RootCAs: opts.roots}) + c, err := New(ctx, log.Logger, clientConn, opts.tlsMode, opts.tlsPKIX, localhost, opts.tlsHostname, Opts{Auth: auths, RootCAs: opts.roots}) if (err == nil) != (expClientErr == nil) || err != nil && !errors.As(err, reflect.New(reflect.ValueOf(expClientErr).Type()).Interface()) && !errors.Is(err, expClientErr) { fail("new client: got err %v, expected %#v", err, expClientErr) } @@ -382,13 +384,13 @@ test func TestErrors(t *testing.T) { ctx := context.Background() - log := mlog.New("") + log := mlog.New("smtpclient", nil) // Invalid greeting. run(t, func(s xserver) { s.writeline("bogus") // Invalid, should be "220 ". }, func(conn net.Conn) { - _, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) var xerr Error if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent { panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err)) @@ -399,7 +401,7 @@ func TestErrors(t *testing.T) { run(t, func(s xserver) { s.conn.Close() }, func(conn net.Conn) { - _, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) var xerr Error if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) || !errors.As(err, &xerr) || xerr.Permanent { panic(fmt.Errorf("got %#v (%v), expected ErrUnexpectedEOF without Permanent", err, err)) @@ -410,7 +412,7 @@ func TestErrors(t *testing.T) { run(t, func(s xserver) { s.writeline("521 not accepting connections") }, func(conn net.Conn) { - _, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) var xerr Error if err == nil || !errors.Is(err, ErrStatus) || !errors.As(err, &xerr) || !xerr.Permanent { panic(fmt.Errorf("got %#v, expected ErrStatus with Permanent", err)) @@ -421,7 +423,7 @@ func TestErrors(t *testing.T) { run(t, func(s xserver) { s.writeline("2200 mox.example") // Invalid, too many digits. }, func(conn net.Conn) { - _, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) var xerr Error if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent { panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err)) @@ -435,7 +437,7 @@ func TestErrors(t *testing.T) { s.writeline("250-mox.example") s.writeline("500 different code") // Invalid. }, func(conn net.Conn) { - _, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + _, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) var xerr Error if err == nil || !errors.Is(err, ErrProtocol) || !errors.As(err, &xerr) || xerr.Permanent { panic(fmt.Errorf("got %#v, expected ErrProtocol without Permanent", err)) @@ -451,7 +453,7 @@ func TestErrors(t *testing.T) { s.readline("MAIL FROM:") s.writeline("550 5.7.0 not allowed") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) if err != nil { panic(err) } @@ -471,7 +473,7 @@ func TestErrors(t *testing.T) { s.readline("MAIL FROM:") s.writeline("451 bad sender") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) if err != nil { panic(err) } @@ -493,7 +495,7 @@ func TestErrors(t *testing.T) { s.readline("RCPT TO:") s.writeline("451") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) if err != nil { panic(err) } @@ -517,7 +519,7 @@ func TestErrors(t *testing.T) { s.readline("DATA") s.writeline("550 no!") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) if err != nil { panic(err) } @@ -537,7 +539,7 @@ func TestErrors(t *testing.T) { s.readline("STARTTLS") s.writeline("502 command not implemented") }, func(conn net.Conn) { - _, err := New(ctx, log, conn, TLSRequiredStartTLS, true, localhost, dns.Domain{ASCII: "mox.example"}, Opts{}) + _, err := New(ctx, log.Logger, conn, TLSRequiredStartTLS, true, localhost, dns.Domain{ASCII: "mox.example"}, Opts{}) var xerr Error if err == nil || !errors.Is(err, ErrTLS) || !errors.As(err, &xerr) || !xerr.Permanent { panic(fmt.Errorf("got %#v, expected ErrTLS with Permanent", err)) @@ -553,7 +555,7 @@ func TestErrors(t *testing.T) { s.readline("MAIL FROM:") s.writeline("451 enough") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSSkip, false, localhost, dns.Domain{ASCII: "mox.example"}, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSSkip, false, localhost, dns.Domain{ASCII: "mox.example"}, Opts{}) if err != nil { panic(err) } @@ -583,7 +585,7 @@ func TestErrors(t *testing.T) { s.readline("DATA") s.writeline("550 not now") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) if err != nil { panic(err) } @@ -613,7 +615,7 @@ func TestErrors(t *testing.T) { s.readline("MAIL FROM:") s.writeline("550 ok") }, func(conn net.Conn) { - c, err := New(ctx, log, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) + c, err := New(ctx, log.Logger, conn, TLSOpportunistic, false, localhost, zerohost, Opts{}) if err != nil { panic(err) } diff --git a/smtpclient/dial.go b/smtpclient/dial.go index 95626a4..55dd591 100644 --- a/smtpclient/dial.go +++ b/smtpclient/dial.go @@ -6,6 +6,8 @@ import ( "net" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" @@ -51,7 +53,8 @@ type Dialer interface { // If we have fully specified local SMTP listener IPs, we set those for the // outgoing connection. The admin probably configured these same IPs in SPF, but // others possibly not. -func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) { +func Dial(ctx context.Context, elog *slog.Logger, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) { + log := mlog.New("smtpclient", elog) timeout := 30 * time.Second if deadline, ok := ctx.Deadline(); ok && len(ips) > 0 { timeout = time.Until(deadline) / time.Duration(len(ips)) @@ -61,7 +64,7 @@ func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain, var lastIP net.IP for _, ip := range ips { addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port)) - log.Debug("dialing host", mlog.Field("addr", addr)) + log.Debug("dialing host", slog.String("addr", addr)) var laddr net.Addr for _, lip := range mox.Conf.Static.SpecifiedSMTPListenIPs { ipIs4 := ip.To4() != nil @@ -73,12 +76,12 @@ func Dial(ctx context.Context, log *mlog.Log, dialer Dialer, host dns.IPDomain, } conn, err := dial(ctx, dialer, timeout, addr, laddr) if err == nil { - log.Debug("connected to host", mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr)) + log.Debug("connected to host", slog.Any("host", host), slog.String("addr", addr), slog.Any("laddr", laddr)) name := host.String() dialedIPs[name] = append(dialedIPs[name], ip) return conn, ip, nil } - log.Debugx("connection attempt", err, mlog.Field("host", host), mlog.Field("addr", addr), mlog.Field("laddr", laddr)) + log.Debugx("connection attempt", err, slog.Any("host", host), slog.String("addr", addr), slog.Any("laddr", laddr)) lastErr = err lastIP = ip } diff --git a/smtpclient/dial_test.go b/smtpclient/dial_test.go index 97f3eae..b660bc6 100644 --- a/smtpclient/dial_test.go +++ b/smtpclient/dial_test.go @@ -14,7 +14,7 @@ import ( func TestDialHost(t *testing.T) { // We mostly want to test that dialing a second time switches to the other address family. ctxbg := context.Background() - log := mlog.New("smtpclient") + log := mlog.New("smtpclient", nil) resolver := dns.MockResolver{ A: map[string][]string{ @@ -37,20 +37,20 @@ func TestDialHost(t *testing.T) { } dialedIPs := map[string][]net.IP{} - _, _, _, ips, dualstack, err := GatherIPs(ctxbg, log, resolver, ipdomain("dualstack.example"), dialedIPs) + _, _, _, ips, dualstack, err := GatherIPs(ctxbg, log.Logger, resolver, ipdomain("dualstack.example"), dialedIPs) if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}) || !dualstack { t.Fatalf("expected err nil, address 10.0.0.1,2001:db8::1, dualstack true, got %v %v %v", err, ips, dualstack) } - _, ip, err := Dial(ctxbg, log, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) + _, ip, err := Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) if err != nil || ip.String() != "10.0.0.1" { t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack) } - _, _, _, ips, dualstack, err = GatherIPs(ctxbg, log, resolver, ipdomain("dualstack.example"), dialedIPs) + _, _, _, ips, dualstack, err = GatherIPs(ctxbg, log.Logger, resolver, ipdomain("dualstack.example"), dialedIPs) if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("10.0.0.1")}) || !dualstack { t.Fatalf("expected err nil, address 2001:db8::1,10.0.0.1, dualstack true, got %v %v %v", err, ips, dualstack) } - _, ip, err = Dial(ctxbg, log, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) + _, ip, err = Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) if err != nil || ip.String() != "2001:db8::1" { t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack) } diff --git a/smtpclient/gather.go b/smtpclient/gather.go index 8807cc9..cfb5c48 100644 --- a/smtpclient/gather.go +++ b/smtpclient/gather.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/adns" "github.com/mjl-/mox/dns" @@ -45,9 +47,11 @@ var ( // were found, both the original and expanded next-hops must be authentic for DANE // to apply. For a non-IP with no MX records found, the authentic result can be // used to decide which of the names to use as TLSA base domain. -func GatherDestinations(ctx context.Context, log *mlog.Log, resolver dns.Resolver, origNextHop dns.IPDomain) (haveMX, origNextHopAuthentic, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, hosts []dns.IPDomain, permanent bool, err error) { +func GatherDestinations(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, origNextHop dns.IPDomain) (haveMX, origNextHopAuthentic, expandedNextHopAuthentic bool, expandedNextHop dns.Domain, hosts []dns.IPDomain, permanent bool, err error) { // ../rfc/5321:3824 + log := mlog.New("smtpclient", elog) + // IP addresses are dialed directly, and don't have TLSA records. if len(origNextHop.IP) > 0 { return false, false, false, expandedNextHop, []dns.IPDomain{origNextHop}, false, nil @@ -167,7 +171,9 @@ func GatherDestinations(ctx context.Context, log *mlog.Log, resolver dns.Resolve // GatherIPs looks up the IPs to try for connecting to host, with the IPs ordered // to take previous attempts into account. For use with DANE, the CNAME-expanded // name is returned, and whether the DNS responses were authentic. -func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host dns.IPDomain, dialedIPs map[string][]net.IP) (authentic bool, expandedAuthentic bool, expandedHost dns.Domain, ips []net.IP, dualstack bool, rerr error) { +func GatherIPs(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, host dns.IPDomain, dialedIPs map[string][]net.IP) (authentic bool, expandedAuthentic bool, expandedHost dns.Domain, ips []net.IP, dualstack bool, rerr error) { + log := mlog.New("smtpclient", elog) + if len(host.IP) > 0 { return false, false, dns.Domain{}, []net.IP{host.IP}, false, nil } @@ -250,7 +256,7 @@ func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host d // Prefer "i" if it is the same as last and we should be preferring it. return preferPrev && ips[i].Equal(prevIP) }) - log.Debug("ordered ips for dialing", mlog.Field("ips", ips)) + log.Debug("ordered ips for dialing", slog.Any("ips", ips)) } return } @@ -268,7 +274,9 @@ func GatherIPs(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host d // must do TLS, but not verify the remote TLS certificate. // // Returned values are always meaningful, also when an error was returned. -func GatherTLSA(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain) (daneRequired bool, daneRecords []adns.TLSA, tlsaBaseDomain dns.Domain, err error) { +func GatherTLSA(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain) (daneRequired bool, daneRecords []adns.TLSA, tlsaBaseDomain dns.Domain, err error) { + log := mlog.New("smtpclient", elog) + // ../rfc/7672:912 // This function is only called when the lookup of host was authentic. @@ -288,32 +296,32 @@ func GatherTLSA(ctx context.Context, log *mlog.Log, resolver dns.Resolver, host } if len(l) == 0 || err != nil { daneRequired = err != nil - log.Debugx("gathering tlsa records failed", err, mlog.Field("danerequired", daneRequired), mlog.Field("basedomain", tlsaBaseDomain)) + log.Debugx("gathering tlsa records failed", err, slog.Bool("danerequired", daneRequired), slog.Any("basedomain", tlsaBaseDomain)) return daneRequired, nil, tlsaBaseDomain, err } daneRequired = len(l) > 0 l = filterUsableTLSARecords(log, l) - log.Debug("tlsa records exist", mlog.Field("danerequired", daneRequired), mlog.Field("records", l), mlog.Field("basedomain", tlsaBaseDomain)) + log.Debug("tlsa records exist", slog.Bool("danerequired", daneRequired), slog.Any("records", l), slog.Any("basedomain", tlsaBaseDomain)) return daneRequired, l, tlsaBaseDomain, err } // lookupTLSACNAME composes a TLSA domain name to lookup, follows CNAMEs and looks // up TLSA records. no TLSA records exist, a nil error is returned as it means // the host does not opt-in to DANE. -func lookupTLSACNAME(ctx context.Context, log *mlog.Log, resolver dns.Resolver, port int, protocol string, host dns.Domain) (l []adns.TLSA, rerr error) { +func lookupTLSACNAME(ctx context.Context, log mlog.Log, resolver dns.Resolver, port int, protocol string, host dns.Domain) (l []adns.TLSA, rerr error) { name := fmt.Sprintf("_%d._%s.%s", port, protocol, host.ASCII+".") for i := 0; ; i++ { cname, result, err := resolver.LookupCNAME(ctx, name) if dns.IsNotFound(err) { if !result.Authentic { - log.Debugx("cname nxdomain result during tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name)) + log.Debugx("cname nxdomain result during tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name)) return nil, nil } break } else if err != nil { return nil, fmt.Errorf("looking up cname for tlsa candidate base domain: %w", err) } else if !result.Authentic { - log.Debugx("cname result during tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name)) + log.Debugx("cname result during tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name)) return nil, nil } if i == 10 { @@ -325,18 +333,18 @@ func lookupTLSACNAME(ctx context.Context, log *mlog.Log, resolver dns.Resolver, var err error l, result, err = resolver.LookupTLSA(ctx, 0, "", name) if dns.IsNotFound(err) || err == nil && len(l) == 0 { - log.Debugx("no tlsa records for host, not doing dane", err, mlog.Field("host", host), mlog.Field("name", name), mlog.Field("authentic", result.Authentic)) + log.Debugx("no tlsa records for host, not doing dane", err, slog.Any("host", host), slog.String("name", name), slog.Bool("authentic", result.Authentic)) return nil, nil } else if err != nil { return nil, fmt.Errorf("looking up tlsa records for tlsa candidate base domain: %w", err) } else if !result.Authentic { - log.Debugx("tlsa lookup not authentic, not doing dane for host", err, mlog.Field("host", host), mlog.Field("name", name)) + log.Debugx("tlsa lookup not authentic, not doing dane for host", err, slog.Any("host", host), slog.String("name", name)) return nil, nil } return l, nil } -func filterUsableTLSARecords(log *mlog.Log, l []adns.TLSA) []adns.TLSA { +func filterUsableTLSARecords(log mlog.Log, l []adns.TLSA) []adns.TLSA { // Gather "usable" records. ../rfc/7672:708 o := 0 for _, r := range l { @@ -368,12 +376,12 @@ func filterUsableTLSARecords(log *mlog.Log, l []adns.TLSA) []adns.TLSA { } case adns.TLSAMatchTypeSHA256: if len(r.CertAssoc) != sha256.Size { - log.Debug("dane tlsa record with wrong data size for sha2-256", mlog.Field("got", len(r.CertAssoc)), mlog.Field("expect", sha256.Size)) + log.Debug("dane tlsa record with wrong data size for sha2-256", slog.Int("got", len(r.CertAssoc)), slog.Int("expect", sha256.Size)) continue } case adns.TLSAMatchTypeSHA512: if len(r.CertAssoc) != sha512.Size { - log.Debug("dane tlsa record with wrong data size for sha2-512", mlog.Field("got", len(r.CertAssoc)), mlog.Field("expect", sha512.Size)) + log.Debug("dane tlsa record with wrong data size for sha2-512", slog.Int("got", len(r.CertAssoc)), slog.Int("expect", sha512.Size)) continue } default: diff --git a/smtpclient/gather_test.go b/smtpclient/gather_test.go index 20b0dcf..37903a2 100644 --- a/smtpclient/gather_test.go +++ b/smtpclient/gather_test.go @@ -47,7 +47,7 @@ func ipdomains(s ...string) (l []dns.IPDomain) { // exist or has temporary error. func TestGatherDestinations(t *testing.T) { ctxbg := context.Background() - log := mlog.New("smtpclient") + log := mlog.New("smtpclient", nil) resolver := dns.MockResolver{ MX: map[string][]*net.MX{ @@ -89,7 +89,7 @@ func TestGatherDestinations(t *testing.T) { test := func(ipd dns.IPDomain, expHosts []dns.IPDomain, expDomain dns.Domain, expPerm, expAuthic, expExpAuthic bool, expErr error) { t.Helper() - _, authic, authicExp, ed, hosts, perm, err := GatherDestinations(ctxbg, log, resolver, ipd) + _, authic, authicExp, ed, hosts, perm, err := GatherDestinations(ctxbg, log.Logger, resolver, ipd) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { // todo: could also check the individual errors? code currently does not have structured errors. t.Fatalf("gather hosts: %v, expected %v", err, expErr) @@ -134,7 +134,7 @@ func TestGatherDestinations(t *testing.T) { func TestGatherIPs(t *testing.T) { ctxbg := context.Background() - log := mlog.New("smtpclient") + log := mlog.New("smtpclient", nil) resolver := dns.MockResolver{ A: map[string][]string{ @@ -164,7 +164,7 @@ func TestGatherIPs(t *testing.T) { test := func(host dns.IPDomain, expAuthic, expAuthicExp bool, expHostExp dns.Domain, expIPs []net.IP, expErr any) { t.Helper() - authic, authicExp, hostExp, ips, _, err := GatherIPs(ctxbg, log, resolver, host, nil) + authic, authicExp, hostExp, ips, _, err := GatherIPs(ctxbg, log.Logger, resolver, host, nil) if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) { // todo: could also check the individual errors? t.Fatalf("gather hosts: %v, expected %v", err, expErr) @@ -207,7 +207,7 @@ func TestGatherIPs(t *testing.T) { func TestGatherTLSA(t *testing.T) { ctxbg := context.Background() - log := mlog.New("smtpclient") + log := mlog.New("smtpclient", nil) record := func(usage, selector, matchType uint8) adns.TLSA { return adns.TLSA{ @@ -253,7 +253,7 @@ func TestGatherTLSA(t *testing.T) { test := func(host dns.Domain, expandedAuthentic bool, expandedHost dns.Domain, expDANERequired bool, expRecords []adns.TLSA, expBaseDom dns.Domain, expErr any) { t.Helper() - daneReq, records, baseDom, err := GatherTLSA(ctxbg, log, resolver, host, expandedAuthentic, expandedHost) + daneReq, records, baseDom, err := GatherTLSA(ctxbg, log.Logger, resolver, host, expandedAuthentic, expandedHost) if (err == nil) != (expErr == nil) || err != nil && !(errors.Is(err, expErr.(error)) || errors.As(err, &expErr)) { // todo: could also check the individual errors? t.Fatalf("gather tlsa: %v, expected %v", err, expErr) diff --git a/smtpserver/alignment.go b/smtpserver/alignment.go index cbb6b1d..870853e 100644 --- a/smtpserver/alignment.go +++ b/smtpserver/alignment.go @@ -5,6 +5,7 @@ import ( "github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/spf" "github.com/mjl-/mox/store" @@ -12,9 +13,9 @@ import ( // Alignment compares the msgFromDomain with the dkim and spf results, and returns // a validation, one of: Strict, Relaxed, None. -func alignment(ctx context.Context, msgFromDomain dns.Domain, dkimResults []dkim.Result, spfStatus spf.Status, spfIdentity *dns.Domain) store.Validation { +func alignment(ctx context.Context, log mlog.Log, msgFromDomain dns.Domain, dkimResults []dkim.Result, spfStatus spf.Status, spfIdentity *dns.Domain) store.Validation { var strict, relaxed bool - msgFromOrgDomain := publicsuffix.Lookup(ctx, msgFromDomain) + msgFromOrgDomain := publicsuffix.Lookup(ctx, log.Logger, msgFromDomain) // todo: should take temperror and permerror into account. for _, dr := range dkimResults { @@ -25,12 +26,12 @@ func alignment(ctx context.Context, msgFromDomain dns.Domain, dkimResults []dkim strict = true break } else { - relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, dr.Sig.Domain) + relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, log.Logger, dr.Sig.Domain) } } if !strict && spfStatus == spf.StatusPass { strict = msgFromDomain == *spfIdentity - relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, *spfIdentity) + relaxed = relaxed || msgFromOrgDomain == publicsuffix.Lookup(ctx, log.Logger, *spfIdentity) } if strict { return store.ValidationStrict diff --git a/smtpserver/analyze.go b/smtpserver/analyze.go index a1193ab..7370544 100644 --- a/smtpserver/analyze.go +++ b/smtpserver/analyze.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/dkim" @@ -89,7 +91,7 @@ func isListDomain(d delivery, ld dns.Domain) bool { return false } -func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis { +func analyze(ctx context.Context, log mlog.Log, resolver dns.Resolver, d delivery) analysis { var headers string mailbox := d.rcptAcc.destination.Mailbox @@ -158,7 +160,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive d.m.MailboxID = mb.ID d.m.MailboxDestinedID = mb.ID } else { - log.Debug("mailbox not found in database", mlog.Field("mailbox", mailbox)) + log.Debug("mailbox not found in database", slog.String("mailbox", mailbox)) } return nil } @@ -206,17 +208,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive if d.dmarcResult.Status != dmarc.StatusPass { log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report") headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n" - } else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { + } else if report, err := dmarcrpt.ParseMessageReport(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { log.Infox("parsing dmarc aggregate report", err) headers += "X-Mox-DMARCReport-Error: could not parse report\r\n" } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil { log.Infox("parsing domain in dmarc aggregate report", err) headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n" } else if _, ok := mox.Conf.Domain(d); !ok { - log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d)) + log.Info("dmarc aggregate report for domain not configured, ignoring", slog.Any("domain", d)) headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n" } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 { - log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0))) + log.Info("dmarc aggregate report with end date in the future, ignoring", slog.Any("domain", d), slog.Time("end", time.Unix(report.ReportMetadata.DateRange.End, 0))) headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n" } else { dmarcReport = report @@ -230,7 +232,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive matchesDomain := func(sigDomain dns.Domain) bool { // RFC seems to require exact DKIM domain match with submitt and message From, we // also allow msgFrom to be subdomain. ../rfc/8460:322 - return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, sigDomain) + return sigDomain == d.msgFrom.Domain || strings.HasSuffix(d.msgFrom.Domain.ASCII, "."+sigDomain.ASCII) && publicsuffix.Lookup(ctx, log.Logger, d.msgFrom.Domain) == publicsuffix.Lookup(ctx, log.Logger, sigDomain) } // Valid DKIM signature for domain must be present. We take "valid" to assume // "passing", not "syntactically valid". We also check for "tlsrpt" as service. @@ -255,13 +257,13 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive if !ok { log.Info("received mail to tlsrpt without acceptable DKIM signature, not processing as tls report") headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n" - } else if report, err := tlsrpt.ParseMessage(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { + } else if report, err := tlsrpt.ParseMessage(log.Logger, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { log.Infox("parsing tls report", err) headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n" } else { var known bool for _, p := range report.Policies { - log.Info("tlsrpt policy domain", mlog.Field("domain", p.Policy.Domain)) + log.Info("tlsrpt policy domain", slog.String("domain", p.Policy.Domain)) if d, err := dns.ParseDomain(p.Policy.Domain); err != nil { log.Infox("parsing domain in tls report", err) } else if _, ok := mox.Conf.Domain(d); ok || d == mox.Conf.Static.HostnameDomain { @@ -297,10 +299,10 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive }) }) if err != nil { - log.Infox("determining reputation", err, mlog.Field("message", d.m)) + log.Infox("determining reputation", err, slog.Any("message", d.m)) return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonReputationError) } - log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method))) + log.Info("reputation analyzed", slog.Bool("conclusive", conclusive), slog.Any("isjunk", isjunk), slog.String("method", string(method))) if conclusive { if !*isjunk { return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason, headers: headers} @@ -340,9 +342,9 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive log.Errorx("get key for verifying subject token", err) return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonSubjectpassError) } - err = subjectpass.Verify(log, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period) + err = subjectpass.Verify(log.Logger, d.dataFile, []byte(subjectpassKey), conf.SubjectPass.Period) pass := err == nil - log.Infox("pass by subject token", err, mlog.Field("pass", pass)) + log.Infox("pass by subject token", err, slog.Bool("pass", pass)) if pass { return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason, headers: headers} } @@ -395,11 +397,11 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive reason = reasonJunkContent if suspiciousIPrevFail && threshold > 0.25 { threshold = 0.25 - log.Info("setting junk threshold due to iprev fail", mlog.Field("threshold", threshold)) + log.Info("setting junk threshold due to iprev fail", slog.Float64("threshold", threshold)) reason = reasonJunkContentStrict } else if !d.tls && threshold > 0.25 { threshold = 0.25 - log.Info("setting junk threshold due to plaintext smtp", mlog.Field("threshold", threshold)) + log.Info("setting junk threshold due to plaintext smtp", slog.Float64("threshold", threshold)) reason = reasonJunkContentStrict } else if (rs == nil || !rs.IsForward) && threshold > 0.25 && !rcptToMatch(d.msgTo) && !rcptToMatch(d.msgCc) { // A common theme in junk messages is your recipient address not being in the To/Cc @@ -407,12 +409,12 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive // providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be // sent with matching Bcc headers. We don't get here for known senders. threshold = 0.25 - log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", mlog.Field("threshold", threshold)) + log.Info("setting junk threshold due to smtp rcpt to and message to/cc address mismatch", slog.Float64("threshold", threshold)) reason = reasonJunkContentStrict } accept = contentProb <= threshold junkSubjectpass = contentProb < threshold-0.2 - log.Info("content analyzed", mlog.Field("accept", accept), mlog.Field("contentprob", contentProb), mlog.Field("subjectpass", junkSubjectpass)) + log.Info("content analyzed", slog.Bool("accept", accept), slog.Float64("contentprob", contentProb), slog.Bool("subjectpass", junkSubjectpass)) } else if err != store.ErrNoJunkFilter { log.Errorx("open junkfilter", err) return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkFilterError) @@ -426,18 +428,18 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive blocked := func(zone dns.Domain) bool { dnsblctx, dnsblcancel := context.WithTimeout(ctx, 30*time.Second) defer dnsblcancel() - if !checkDNSBLHealth(dnsblctx, resolver, zone) { - log.Info("dnsbl not healthy, skipping", mlog.Field("zone", zone)) + if !checkDNSBLHealth(dnsblctx, log, resolver, zone) { + log.Info("dnsbl not healthy, skipping", slog.Any("zone", zone)) return false } - status, expl, err := dnsbl.Lookup(dnsblctx, resolver, zone, net.ParseIP(d.m.RemoteIP)) + status, expl, err := dnsbl.Lookup(dnsblctx, log.Logger, resolver, zone, net.ParseIP(d.m.RemoteIP)) dnsblcancel() if status == dnsbl.StatusFail { - log.Info("rejecting due to listing in dnsbl", mlog.Field("zone", zone), mlog.Field("explanation", expl)) + log.Info("rejecting due to listing in dnsbl", slog.Any("zone", zone), slog.String("explanation", expl)) return true } else if err != nil { - log.Infox("dnsbl lookup", err, mlog.Field("zone", zone), mlog.Field("status", status)) + log.Infox("dnsbl lookup", err, slog.Any("zone", zone), slog.Any("status", status)) } return false } @@ -459,7 +461,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) { log.Info("permanent reject with subjectpass hint of moderately spammy email without reputation") - pass := subjectpass.Generate(d.msgFrom, []byte(subjectpassKey), time.Now()) + pass := subjectpass.Generate(log.Logger, d.msgFrom, []byte(subjectpassKey), time.Now()) return reject(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, subjectpass.Explanation+pass, nil, reasonGiveSubjectpass) } diff --git a/smtpserver/dnsbl.go b/smtpserver/dnsbl.go index 0f732b6..671e9fc 100644 --- a/smtpserver/dnsbl.go +++ b/smtpserver/dnsbl.go @@ -8,6 +8,7 @@ import ( "github.com/mjl-/mox/dns" "github.com/mjl-/mox/dnsbl" + "github.com/mjl-/mox/mlog" ) var dnsblHealth = struct { @@ -23,12 +24,12 @@ type dnsblStatus struct { } // checkDNSBLHealth checks healthiness of DNSBL "zone", keeping the result cached for 4 hours. -func checkDNSBLHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rok bool) { +func checkDNSBLHealth(ctx context.Context, log mlog.Log, resolver dns.Resolver, zone dns.Domain) (rok bool) { dnsblHealth.Lock() defer dnsblHealth.Unlock() status, ok := dnsblHealth.zones[zone] if !ok || time.Since(status.last) > 4*time.Hour { - status.err = dnsbl.CheckHealth(ctx, resolver, zone) + status.err = dnsbl.CheckHealth(ctx, log.Logger, resolver, zone) status.last = time.Now() dnsblHealth.zones[zone] = status } diff --git a/smtpserver/dsn.go b/smtpserver/dsn.go index 56e8396..98fee1d 100644 --- a/smtpserver/dsn.go +++ b/smtpserver/dsn.go @@ -24,7 +24,7 @@ func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, req } } - f, err := store.CreateMessageTemp("smtp-dsn") + f, err := store.CreateMessageTemp(c.log, "smtp-dsn") if err != nil { return fmt.Errorf("creating temp file: %w", err) } diff --git a/smtpserver/fuzz_test.go b/smtpserver/fuzz_test.go index 9c2e1fb..d276d72 100644 --- a/smtpserver/fuzz_test.go +++ b/smtpserver/fuzz_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/queue" "github.com/mjl-/mox/store" @@ -30,17 +31,18 @@ func FuzzServer(f *testing.F) { f.Add("NOOP") f.Add("QUIT") + log := mlog.New("smtpserver", nil) mox.Context = ctxbg mox.ConfigStaticPath = filepath.FromSlash("../testdata/smtpserverfuzz/mox.conf") mox.MustLoadConfig(true, false) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) os.RemoveAll(dataDir) - acc, err := store.OpenAccount("mjl") + acc, err := store.OpenAccount(log, "mjl") if err != nil { f.Fatalf("open account: %v", err) } defer acc.Close() - err = acc.SetPassword("testtest") + err = acc.SetPassword(log, "testtest") if err != nil { f.Fatalf("set password: %v", err) } diff --git a/smtpserver/rejects.go b/smtpserver/rejects.go index fd53c17..2ac5bf7 100644 --- a/smtpserver/rejects.go +++ b/smtpserver/rejects.go @@ -8,6 +8,8 @@ import ( "io" "os" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/message" @@ -17,15 +19,15 @@ import ( ) // rejectPresent returns whether the message is already present in the rejects mailbox. -func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *store.Message, f *os.File) (present bool, msgID string, hash []byte, rerr error) { - if p, err := message.Parse(log, false, store.FileMsgReader(m.MsgPrefix, f)); err != nil { +func rejectPresent(log mlog.Log, acc *store.Account, rejectsMailbox string, m *store.Message, f *os.File) (present bool, msgID string, hash []byte, rerr error) { + if p, err := message.Parse(log.Logger, false, store.FileMsgReader(m.MsgPrefix, f)); err != nil { log.Infox("parsing reject message for message-id", err) } else if header, err := p.Header(); err != nil { log.Infox("parsing reject message header for message-id", err) } else { msgID, _, err = message.MessageIDCanonical(header.Get("Message-Id")) if err != nil { - log.Debugx("parsing message-id for reject", err, mlog.Field("messageid", header.Get("Message-Id"))) + log.Debugx("parsing message-id for reject", err, slog.String("messageid", header.Get("Message-Id"))) } } diff --git a/smtpserver/reputation.go b/smtpserver/reputation.go index 8083bf5..d9429f6 100644 --- a/smtpserver/reputation.go +++ b/smtpserver/reputation.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/mlog" @@ -96,7 +98,7 @@ const ( // ../rfc/6376:1915 // ../rfc/6376:3716 // ../rfc/7208:2167 -func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rconclusive bool, rmethod reputationMethod, rerr error) { +func reputation(tx *bstore.Tx, log mlog.Log, m *store.Message) (rjunk *bool, rconclusive bool, rmethod reputationMethod, rerr error) { boolptr := func(v bool) *bool { return &v } @@ -141,7 +143,7 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc xmessageList := func(q *bstore.Query[store.Message], descr string) []store.Message { t0 := time.Now() l, err := q.List() - log.Debugx("querying messages for reputation", err, mlog.Field("msgs", len(l)), mlog.Field("descr", descr), mlog.Field("queryduration", time.Since(t0))) + log.Debugx("querying messages for reputation", err, slog.Int("msgs", len(l)), slog.String("descr", descr), slog.Duration("queryduration", time.Since(t0))) if err != nil { panic(queryError(fmt.Sprintf("listing messages: %v", err))) } diff --git a/smtpserver/reputation_test.go b/smtpserver/reputation_test.go index b39e2f4..fff888d 100644 --- a/smtpserver/reputation_test.go +++ b/smtpserver/reputation_test.go @@ -11,11 +11,14 @@ import ( "github.com/mjl-/bstore" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/smtp" "github.com/mjl-/mox/store" ) +var pkglog = mlog.New("smtpserver", nil) + func TestReputation(t *testing.T) { boolptr := func(v bool) *bool { return &v @@ -26,6 +29,8 @@ func TestReputation(t *testing.T) { now := time.Now() var uidgen store.UID + log := mlog.New("smtpserver", nil) + message := func(junk bool, ageDays int, ehlo, mailfrom, msgfrom, rcptto string, msgfromvalidation store.Validation, dkimDomains []string, mailfromValid, ehloValid bool, ip string) store.Message { mailFromValidation := store.ValidationNone @@ -77,7 +82,7 @@ func TestReputation(t *testing.T) { MsgFromLocalpart: msgFrom.Localpart, MsgFromDomain: msgFrom.Domain.Name(), - MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, msgFrom.Domain).Name(), + MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, log.Logger, msgFrom.Domain).Name(), MailFromValidated: mailfromValid, EHLOValidated: ehloValid, @@ -119,7 +124,7 @@ func TestReputation(t *testing.T) { rcptToDomain, err := dns.ParseDomain(hm.RcptToDomain) tcheck(t, err, "parse rcptToDomain") - rcptToOrgDomain := publicsuffix.Lookup(ctxbg, rcptToDomain) + rcptToOrgDomain := publicsuffix.Lookup(ctxbg, log.Logger, rcptToDomain) r := store.Recipient{MessageID: hm.ID, Localpart: hm.RcptToLocalpart, Domain: hm.RcptToDomain, OrgDomain: rcptToOrgDomain.Name(), Sent: hm.Received} err = tx.Insert(&r) tcheck(t, err, "insert recipient") @@ -136,7 +141,7 @@ func TestReputation(t *testing.T) { var method reputationMethod err = db.Read(ctxbg, func(tx *bstore.Tx) error { var err error - isjunk, conclusive, method, err = reputation(tx, xlog, &m) + isjunk, conclusive, method, err = reputation(tx, pkglog, &m) return err }) tcheck(t, err, "read tx") diff --git a/smtpserver/server.go b/smtpserver/server.go index b4da960..df92af8 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -27,6 +27,7 @@ import ( "time" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -57,10 +58,6 @@ import ( "github.com/mjl-/mox/tlsrptdb" ) -// Most logging should be done through conn.log* functions. -// Only use log in contexts without connection. -var xlog = mlog.New("smtpserver") - // We use panic and recover for error handling while executing commands. // These errors signal the connection must be closed. var errIO = errors.New("io error") @@ -233,14 +230,15 @@ func Listen() { var servers []func() func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig *tls.Config, submission, xtls bool, maxMessageSize int64, requireTLSForAuth, requireTLSForDelivery, requireTLS bool, dnsBLs []dns.Domain, firstTimeSenderDelay time.Duration) { + log := mlog.New("smtpserver", nil) addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) if os.Getuid() == 0 { - xlog.Print("listening for smtp", mlog.Field("listener", name), mlog.Field("address", addr), mlog.Field("protocol", protocol)) + log.Print("listening for smtp", slog.String("listener", name), slog.String("address", addr), slog.String("protocol", protocol)) } network := mox.Network(ip) ln, err := mox.Listen(network, addr) if err != nil { - xlog.Fatalx("smtp: listen for smtp", err, mlog.Field("protocol", protocol), mlog.Field("listener", name)) + log.Fatalx("smtp: listen for smtp", err, slog.String("protocol", protocol), slog.String("listener", name)) } if xtls { ln = tls.NewListener(ln, tlsConfig) @@ -250,10 +248,12 @@ func listen1(protocol, name, ip string, port int, hostname dns.Domain, tlsConfig for { conn, err := ln.Accept() if err != nil { - xlog.Infox("smtp: accept", err, mlog.Field("protocol", protocol), mlog.Field("listener", name)) + log.Infox("smtp: accept", err, slog.String("protocol", protocol), slog.String("listener", name)) continue } - resolver := dns.StrictResolver{} // By leaving Pkg empty, it'll be set by each package that uses the resolver, e.g. spf/dkim/dmarc. + + // Package is set on the resolver by the dkim/spf/dmarc/etc packages. + resolver := dns.StrictResolver{Log: log.Logger} go serve(name, mox.Cid(), hostname, tlsConfig, conn, resolver, submission, xtls, maxMessageSize, requireTLSForAuth, requireTLSForDelivery, requireTLS, dnsBLs, firstTimeSenderDelay) } } @@ -292,7 +292,7 @@ type conn struct { localIP net.IP remoteIP net.IP hostname dns.Domain - log *mlog.Log + log mlog.Log maxMessageSize int64 requireTLSForAuth bool requireTLSForDelivery bool // If set, delivery is only allowed with TLS (STARTTLS), except if delivery is to a TLS reporting address. @@ -378,7 +378,7 @@ func (c *conn) xcheckAuth() { } } -func (c *conn) xtrace(level mlog.Level) func() { +func (c *conn) xtrace(level slog.Level) func() { c.xflush() c.tr.SetTrace(level) c.tw.SetTrace(level) @@ -458,7 +458,7 @@ func (c *conn) Read(buf []byte) (int, error) { var bufpool = moxio.NewBufpool(8, 2*1024) func (c *conn) readline() string { - line, err := bufpool.Readline(c.r) + line, err := bufpool.Readline(c.log, c.r) if err != nil && errors.Is(err, moxio.ErrLineTooLong) { c.writecodeline(smtp.C500BadSyntax, smtp.SeProto5Other0, "line too long, smtp max is 512, we reached 2048", nil) panic(fmt.Errorf("%s (%w)", err, errIO)) @@ -476,7 +476,7 @@ func (c *conn) bwritecodeline(code int, secode string, msg string, err error) { ecode = fmt.Sprintf("%d.%s", code/100, secode) } metricCommands.WithLabelValues(c.kind(), c.cmd, fmt.Sprintf("%d", code), ecode).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) - c.log.Debugx("smtp command result", err, mlog.Field("kind", c.kind()), mlog.Field("cmd", c.cmd), mlog.Field("code", fmt.Sprintf("%d", code)), mlog.Field("ecode", ecode), mlog.Field("duration", time.Since(c.cmdStart))) + c.log.Debugx("smtp command result", err, slog.String("kind", c.kind()), slog.String("cmd", c.cmd), slog.Int("code", code), slog.String("ecode", ecode), slog.Duration("duration", time.Since(c.cmdStart))) var sep string if ecode != "" { @@ -563,15 +563,18 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C dnsBLs: dnsBLs, firstTimeSenderDelay: firstTimeSenderDelay, } - c.log = xlog.MoreFields(func() []mlog.Pair { + var logmutex sync.Mutex + c.log = mlog.New("smtpserver", nil).WithFunc(func() []slog.Attr { + logmutex.Lock() + defer logmutex.Unlock() now := time.Now() - l := []mlog.Pair{ - mlog.Field("cid", c.cid), - mlog.Field("delta", now.Sub(c.lastlog)), + l := []slog.Attr{ + slog.Int64("cid", c.cid), + slog.Duration("delta", now.Sub(c.lastlog)), } c.lastlog = now if c.username != "" { - l = append(l, mlog.Field("username", c.username)) + l = append(l, slog.String("username", c.username)) } return l }) @@ -581,7 +584,7 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C c.w = bufio.NewWriter(c.tw) metricConnection.WithLabelValues(c.kind()).Inc() - c.log.Info("new connection", mlog.Field("remote", c.conn.RemoteAddr()), mlog.Field("local", c.conn.LocalAddr()), mlog.Field("submission", submission), mlog.Field("tls", tls), mlog.Field("listener", listenerName)) + c.log.Info("new connection", slog.Any("remote", c.conn.RemoteAddr()), slog.Any("local", c.conn.LocalAddr()), slog.Bool("submission", submission), slog.Bool("tls", tls), slog.String("listener", listenerName)) defer func() { c.origConn.Close() // Close actual TCP socket, regardless of TLS on top. @@ -599,7 +602,7 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C } else if err, ok := x.(error); ok && isClosed(err) { c.log.Infox("connection closed", err) } else { - c.log.Error("unhandled panic", mlog.Field("err", x)) + c.log.Error("unhandled panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Smtpserver) } @@ -621,13 +624,13 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C // If remote IP/network resulted in too many authentication failures, refuse to serve. if submission && !mox.LimiterFailedAuth.CanAdd(c.remoteIP, time.Now(), 1) { metrics.AuthenticationRatelimitedInc("submission") - c.log.Debug("refusing connection due to many auth failures", mlog.Field("remoteip", c.remoteIP)) + c.log.Debug("refusing connection due to many auth failures", slog.Any("remoteip", c.remoteIP)) c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many auth failures", nil) return } if !limiterConnections.Add(c.remoteIP, time.Now(), 1) { - c.log.Debug("refusing connection due to many open connections", mlog.Field("remoteip", c.remoteIP)) + c.log.Debug("refusing connection due to many open connections", slog.Any("remoteip", c.remoteIP)) c.writecodeline(smtp.C421ServiceUnavail, smtp.SePol7Other0, "too many open connections from your ip or network", nil) return } @@ -892,7 +895,7 @@ func (c *conn) cmdStarttls(p *parser) { } cancel() tlsversion, ciphersuite := mox.TLSInfo(tlsConn) - c.log.Debug("tls server handshake done", mlog.Field("tls", tlsversion), mlog.Field("ciphersuite", ciphersuite)) + c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite)) c.conn = tlsConn c.tr = moxio.NewTraceReader(c.log, "RC: ", c) c.tw = moxio.NewTraceWriter(c.log, "LS: ", c) @@ -1030,11 +1033,11 @@ func (c *conn) cmdAuth(p *parser) { xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "cannot assume other role") } - acc, err := store.OpenEmailAuth(authc, password) + acc, err := store.OpenEmailAuth(c.log, authc, password) if err != nil && errors.Is(err, store.ErrUnknownCredentials) { // ../rfc/4954:274 authResult = "badcreds" - c.log.Info("failed authentication attempt", mlog.Field("username", authc), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", authc), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") } xcheckf(err, "verifying credentials") @@ -1075,11 +1078,11 @@ func (c *conn) cmdAuth(p *parser) { password := string(xreadContinuation()) c.xtrace(mlog.LevelTrace) // Restore. - acc, err := store.OpenEmailAuth(username, password) + acc, err := store.OpenEmailAuth(c.log, username, password) if err != nil && errors.Is(err, store.ErrUnknownCredentials) { // ../rfc/4954:274 authResult = "badcreds" - c.log.Info("failed authentication attempt", mlog.Field("username", username), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", username), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") } xcheckf(err, "verifying credentials") @@ -1107,11 +1110,11 @@ func (c *conn) cmdAuth(p *parser) { xsmtpUserErrorf(smtp.C501BadParamSyntax, smtp.SeProto5BadParams4, "malformed cram-md5 response") } addr := t[0] - c.log.Debug("cram-md5 auth", mlog.Field("address", addr)) - acc, _, err := store.OpenEmail(addr) + c.log.Debug("cram-md5 auth", slog.String("address", addr)) + acc, _, err := store.OpenEmail(c.log, addr) if err != nil { if errors.Is(err, store.ErrUnknownCredentials) { - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") } } @@ -1127,7 +1130,7 @@ func (c *conn) cmdAuth(p *parser) { err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error { password, err := bstore.QueryTx[store.Password](tx).Get() if err == bstore.ErrAbsent { - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") } if err != nil { @@ -1141,8 +1144,8 @@ func (c *conn) cmdAuth(p *parser) { xcheckf(err, "tx read") }) if ipadhash == nil || opadhash == nil { - c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", mlog.Field("username", addr)) - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("cram-md5 auth attempt without derived secrets set, save password again to store secrets", slog.String("username", addr)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") } @@ -1151,7 +1154,7 @@ func (c *conn) cmdAuth(p *parser) { opadhash.Write(ipadhash.Sum(nil)) digest := fmt.Sprintf("%x", opadhash.Sum(nil)) if digest != t[1] { - c.log.Info("failed authentication attempt", mlog.Field("username", addr), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", addr), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass") } @@ -1181,13 +1184,13 @@ func (c *conn) cmdAuth(p *parser) { c0 := xreadInitial() ss, err := scram.NewServer(h, c0) xcheckf(err, "starting scram") - c.log.Debug("scram auth", mlog.Field("authentication", ss.Authentication)) - acc, _, err := store.OpenEmail(ss.Authentication) + c.log.Debug("scram auth", slog.String("authentication", ss.Authentication)) + acc, _, err := store.OpenEmail(c.log, ss.Authentication) if err != nil { // todo: we could continue scram with a generated salt, deterministically generated // from the username. that way we don't have to store anything but attackers cannot // learn if an account exists. same for absent scram saltedpassword below. - c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible") } defer func() { @@ -1209,8 +1212,8 @@ func (c *conn) cmdAuth(p *parser) { xscram = password.SCRAMSHA256 } if err == bstore.ErrAbsent || err == nil && (len(xscram.Salt) == 0 || xscram.Iterations == 0 || len(xscram.SaltedPassword) == 0) { - c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", mlog.Field("address", ss.Authentication)) - c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) + c.log.Info("scram auth attempt without derived secrets set, save password again to store secrets", slog.String("address", ss.Authentication)) + c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C454TempAuthFail, smtp.SeSys3Other0, "scram not possible") } xcheckf(err, "fetching credentials") @@ -1230,7 +1233,7 @@ func (c *conn) cmdAuth(p *parser) { c.readline() // Should be "*" for cancellation. if errors.Is(err, scram.ErrInvalidProof) { authResult = "badcreds" - c.log.Info("failed authentication attempt", mlog.Field("username", ss.Authentication), mlog.Field("remote", c.remoteIP)) + c.log.Info("failed authentication attempt", slog.String("username", ss.Authentication), slog.Any("remote", c.remoteIP)) xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad credentials") } xcheckf(err, "server final") @@ -1398,11 +1401,11 @@ func (c *conn) cmdMail(p *parser) { if c.submission && (len(rpath.IPDomain.IP) > 0 || !rpathAllowed()) { // ../rfc/6409:522 - c.log.Info("submission with unconfigured mailfrom", mlog.Field("user", c.username), mlog.Field("mailfrom", rpath.String())) + c.log.Info("submission with unconfigured mailfrom", slog.String("user", c.username), slog.String("mailfrom", rpath.String())) xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user") } else if !c.submission && len(rpath.IPDomain.IP) > 0 { // todo future: allow if the IP is the same as this connection is coming from? does later code allow this? - c.log.Info("delivery from address without domain", mlog.Field("mailfrom", rpath.String())) + c.log.Info("delivery from address without domain", slog.String("mailfrom", rpath.String())) xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7Other0, "domain name required") } @@ -1494,7 +1497,7 @@ func (c *conn) cmdRcpt(p *parser) { cidctx := context.WithValue(mox.Context, mlog.CidKey, c.cid) spfctx, spfcancel := context.WithTimeout(cidctx, time.Minute) defer spfcancel() - receivedSPF, _, _, _, err := spf.Verify(spfctx, c.resolver, spfArgs) + receivedSPF, _, _, _, err := spf.Verify(spfctx, c.log.Logger, c.resolver, spfArgs) spfcancel() if err != nil { c.log.Errorx("spf verify for multiple recipients", err) @@ -1542,7 +1545,7 @@ func (c *conn) cmdRcpt(p *parser) { // note: not local for !c.submission is the signal this address is in error. c.recipients = append(c.recipients, rcptAccount{fpath, false, "", config.Destination{}, ""}) } else { - c.log.Errorx("looking up account for delivery", err, mlog.Field("rcptto", fpath)) + c.log.Errorx("looking up account for delivery", err, slog.Any("rcptto", fpath)) xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "error processing") } c.bwritecodeline(smtp.C250Completed, smtp.SeAddr1Other0, "now on the list", nil) @@ -1583,7 +1586,7 @@ func (c *conn) cmdData(p *parser) { defer c.xtrace(mlog.LevelTracedata)() // We read the data into a temporary file. We limit the size and do basic analysis while reading. - dataFile, err := store.CreateMessageTemp("smtp-deliver") + dataFile, err := store.CreateMessageTemp(c.log, "smtp-deliver") if err != nil { xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "creating temporary file for message: %s", err) } @@ -1631,9 +1634,9 @@ func (c *conn) cmdData(p *parser) { if Localserve && moxvar.Pedantic { // Require that message can be parsed fully. - p, err := message.Parse(c.log, false, dataFile) + p, err := message.Parse(c.log.Logger, false, dataFile) if err == nil { - err = p.Walk(c.log, nil) + err = p.Walk(c.log.Logger, nil) } if err != nil { // ../rfc/6409:541 @@ -1665,9 +1668,9 @@ func (c *conn) cmdData(p *parser) { iprevStatus, revName, revNames, iprevAuthentic, err = iprev.Lookup(iprevctx, c.resolver, c.remoteIP) iprevcancel() if err != nil { - c.log.Infox("reverse-forward lookup", err, mlog.Field("remoteip", c.remoteIP)) + c.log.Infox("reverse-forward lookup", err, slog.Any("remoteip", c.remoteIP)) } - c.log.Debug("dns iprev check", mlog.Field("addr", c.remoteIP), mlog.Field("status", iprevStatus)) + c.log.Debug("dns iprev check", slog.Any("addr", c.remoteIP), slog.Any("status", iprevStatus)) var name string if revName != "" { name = revName @@ -1721,7 +1724,7 @@ func (c *conn) cmdData(p *parser) { recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "via", "tcp", "with", with+withComment, "id", mox.ReceivedID(c.cid)) // ../rfc/5321:3158 if c.tls { tlsConn := c.conn.(*tls.Conn) - tlsComment := message.TLSReceivedComment(c.log, tlsConn.ConnectionState()) + tlsComment := mox.TLSReceivedComment(c.log, tlsConn.ConnectionState()) recvHdr.Add(" ", tlsComment...) } recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z)) @@ -1765,10 +1768,10 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr // for other users. // We don't check the Sender field, there is no expectation of verification, ../rfc/7489:2948 // and with Resent headers it seems valid to have someone else as Sender. ../rfc/5322:1578 - msgFrom, _, header, err := message.From(c.log, true, dataFile) + msgFrom, _, header, err := message.From(c.log.Logger, true, dataFile) if err != nil { metricSubmission.WithLabelValues("badmessage").Inc() - c.log.Infox("parsing message From address", err, mlog.Field("user", c.username)) + c.log.Infox("parsing message From address", err, slog.String("user", c.username)) xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SeMsg6Other0, "cannot parse header or From address: %v", err) } accName, _, _, err := mox.FindAccount(msgFrom.Localpart, msgFrom.Domain, true) @@ -1778,7 +1781,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr err = mox.ErrAccountNotFound } metricSubmission.WithLabelValues("badfrom").Inc() - c.log.Infox("verifying message From address", err, mlog.Field("user", c.username), mlog.Field("msgfrom", msgFrom)) + c.log.Infox("verifying message From address", err, slog.String("user", c.username), slog.Any("msgfrom", msgFrom)) xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7DeliveryUnauth1, "must match authenticated user") } @@ -1835,16 +1838,16 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr // Add DKIM signatures. confDom, ok := mox.Conf.Domain(msgFrom.Domain) if !ok { - c.log.Error("domain disappeared", mlog.Field("domain", msgFrom.Domain)) + c.log.Error("domain disappeared", slog.Any("domain", msgFrom.Domain)) xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "internal error") } dkimConfig := confDom.DKIM if len(dkimConfig.Sign) > 0 { if canonical, err := mox.CanonicalLocalpart(msgFrom.Localpart, confDom); err != nil { - c.log.Errorx("determining canonical localpart for dkim signing", err, mlog.Field("localpart", msgFrom.Localpart)) - } else if dkimHeaders, err := dkim.Sign(ctx, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil { - c.log.Errorx("dkim sign for domain", err, mlog.Field("domain", msgFrom.Domain)) + c.log.Errorx("determining canonical localpart for dkim signing", err, slog.Any("localpart", msgFrom.Localpart)) + } else if dkimHeaders, err := dkim.Sign(ctx, c.log.Logger, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil { + c.log.Errorx("dkim sign for domain", err, slog.Any("domain", msgFrom.Domain)) metricServerErrors.WithLabelValues("dkimsign").Inc() } else { msgPrefix = append(msgPrefix, []byte(dkimHeaders)...) @@ -1877,7 +1880,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr mox.Sleep(mox.Context, time.Hour) xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out submission due to special localpart") } else if code != 0 { - c.log.Info("failure due to special localpart", mlog.Field("code", code)) + c.log.Info("failure due to special localpart", slog.Int("code", code)) xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code) } } @@ -1894,7 +1897,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err) } metricSubmission.WithLabelValues("ok").Inc() - c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize)) + c.log.Info("message queued for delivery", slog.Any("mailfrom", *c.mailFrom), slog.Any("rcptto", rcptAcc.rcptTo), slog.Bool("smtputf8", c.smtputf8), slog.Int64("msgsize", msgSize)) err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)}) xcheckf(err, "adding outgoing message") @@ -1950,7 +1953,7 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) { mox.Sleep(mox.Context, time.Hour) xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out command due to special localpart") } else if code != 0 { - c.log.Info("failure due to special localpart", mlog.Field("code", code)) + c.log.Info("failure due to special localpart", slog.Int("code", code)) metricDelivery.WithLabelValues("delivererror", "localserve").Inc() xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code) } @@ -1961,7 +1964,7 @@ func (c *conn) xlocalserveError(lp smtp.Localpart) { func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, iprevAuthentic bool, dataFile *os.File) { // todo: in decision making process, if we run into (some) temporary errors, attempt to continue. if we decide to accept, all good. if we decide to reject, we'll make it a temporary reject. - msgFrom, envelope, headers, err := message.From(c.log, false, dataFile) + msgFrom, envelope, headers, err := message.From(c.log.Logger, false, dataFile) if err != nil { c.log.Infox("parsing message for From address", err) } @@ -2016,7 +2019,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW defer func() { x := recover() // Should not happen, but don't take program down if it does. if x != nil { - c.log.Error("dkim verify panic", mlog.Field("err", x)) + c.log.Error("dkim verify panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Dkimverify) } @@ -2029,7 +2032,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW dkimctx, dkimcancel := context.WithTimeout(ctx, time.Minute) defer dkimcancel() // todo future: we could let user configure which dkim headers they require - dkimResults, dkimErr = dkim.Verify(dkimctx, c.resolver, c.smtputf8, dkim.DefaultPolicy, dataFile, ignoreTestMode) + dkimResults, dkimErr = dkim.Verify(dkimctx, c.log.Logger, c.resolver, c.smtputf8, dkim.DefaultPolicy, dataFile, ignoreTestMode) dkimcancel() }() @@ -2053,7 +2056,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW defer func() { x := recover() // Should not happen, but don't take program down if it does. if x != nil { - c.log.Error("spf verify panic", mlog.Field("err", x)) + c.log.Error("spf verify panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Spfverify) } @@ -2061,7 +2064,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW defer wg.Done() spfctx, spfcancel := context.WithTimeout(ctx, time.Minute) defer spfcancel() - receivedSPF, spfDomain, spfExpl, spfAuthentic, spfErr = spf.Verify(spfctx, c.resolver, spfArgs) + receivedSPF, spfDomain, spfExpl, spfAuthentic, spfErr = spf.Verify(spfctx, c.log.Logger, c.resolver, spfArgs) spfcancel() if spfErr != nil { c.log.Infox("spf verify", spfErr) @@ -2080,7 +2083,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } if nunknown == len(c.recipients) { // During RCPT TO we found that the address does not exist. - c.log.Info("deliver attempt to unknown user(s)", mlog.Field("recipients", c.recipients)) + c.log.Info("deliver attempt to unknown user(s)", slog.Any("recipients", c.recipients)) // Crude attempt to slow down someone trying to guess names. Would work better // with connection rate limiter. @@ -2107,7 +2110,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW c.log.Errorx("dkim verify", dkimErr) authResAddDKIM("none", "", dkimErr.Error(), nil) } else if len(dkimResults) == 0 { - c.log.Info("no dkim-signature header", mlog.Field("mailfrom", c.mailFrom)) + c.log.Info("no dkim-signature header", slog.Any("mailfrom", c.mailFrom)) authResAddDKIM("none", "", "no dkim signatures", nil) } for i, r := range dkimResults { @@ -2147,7 +2150,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW errmsg = r.Err.Error() } authResAddDKIM(string(r.Status), comment, errmsg, props) - c.log.Debugx("dkim verification result", r.Err, mlog.Field("index", i), mlog.Field("mailfrom", c.mailFrom), mlog.Field("status", r.Status), mlog.Field("domain", domain), mlog.Field("selector", selector), mlog.Field("identity", identity)) + c.log.Debugx("dkim verification result", r.Err, slog.Int("index", i), slog.Any("mailfrom", c.mailFrom), slog.Any("status", r.Status), slog.Any("domain", domain), slog.Any("selector", selector), slog.Any("identity", identity)) } // Add SPF results to Authentication-Results header. ../rfc/7208:2141 @@ -2182,7 +2185,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW }) switch receivedSPF.Result { case spf.StatusPass: - c.log.Debug("spf pass", mlog.Field("ip", spfArgs.RemoteIP), mlog.Field("mailfromdomain", spfArgs.MailFromDomain.ASCII)) // todo: log the domain that was actually verified. + c.log.Debug("spf pass", slog.Any("ip", spfArgs.RemoteIP), slog.String("mailfromdomain", spfArgs.MailFromDomain.ASCII)) // todo: log the domain that was actually verified. case spf.StatusFail: if spfExpl != "" { // Filter out potentially hostile text. ../rfc/7208:2529 @@ -2202,14 +2205,14 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW if spfExpl == "" { spfExpl = fmt.Sprintf("your ip %s is not on the SPF allowlist for domain %s", spfArgs.RemoteIP, spfDomain.ASCII) } - c.log.Info("spf fail", mlog.Field("explanation", spfExpl)) // todo future: get this to the client. how? in smtp session in case of a reject due to dmarc fail? + c.log.Info("spf fail", slog.String("explanation", spfExpl)) // todo future: get this to the client. how? in smtp session in case of a reject due to dmarc fail? case spf.StatusTemperror: c.log.Infox("spf temperror", spfErr) case spf.StatusPermerror: c.log.Infox("spf permerror", spfErr) case spf.StatusNone, spf.StatusNeutral, spf.StatusSoftfail: default: - c.log.Error("unknown spf status, treating as None/Neutral", mlog.Field("status", receivedSPF.Result)) + c.log.Error("unknown spf status, treating as None/Neutral", slog.Any("status", receivedSPF.Result)) receivedSPF.Result = spf.StatusNone } @@ -2228,7 +2231,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW Result: string(dmarcResult.Status), } } else { - msgFromValidation = alignment(ctx, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity) + msgFromValidation = alignment(ctx, c.log, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity) // We are doing the DMARC evaluation now. But we only store it for inclusion in an // aggregate report when we actually use it. We use an evaluation for each @@ -2241,7 +2244,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute) defer dmarccancel() - dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage) + dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.log.Logger, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage) dmarccancel() var comment string if dmarcResult.RecordAuthentic { @@ -2265,7 +2268,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507 } - c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain)) + c.log.Debug("dmarc verification", slog.Any("result", dmarcResult.Status), slog.Any("domain", msgFrom.Domain)) // Prepare for analyzing content, calculating reputation. ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP) @@ -2304,13 +2307,13 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW var deliverErrors []deliverError addError := func(rcptAcc rcptAccount, code int, secode string, userError bool, errmsg string) { e := deliverError{rcptAcc.rcptTo, code, secode, userError, errmsg} - c.log.Info("deliver error", mlog.Field("rcptto", e.rcptTo), mlog.Field("code", code), mlog.Field("secode", "secode"), mlog.Field("usererror", userError), mlog.Field("errmsg", errmsg)) + c.log.Info("deliver error", slog.Any("rcptto", e.rcptTo), slog.Int("code", code), slog.String("secode", "secode"), slog.Bool("usererror", userError), slog.String("errmsg", errmsg)) deliverErrors = append(deliverErrors, e) } // For each recipient, do final spam analysis and delivery. for _, rcptAcc := range c.recipients { - log := c.log.Fields(mlog.Field("mailfrom", c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo)) + log := c.log.With(slog.Any("mailfrom", c.mailFrom), slog.Any("rcptto", rcptAcc.rcptTo)) // If this is not a valid local user, we send back a DSN. This can only happen when // there are also valid recipients, and only when remote is SPF-verified, so the DSN @@ -2326,9 +2329,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW continue } - acc, err := store.OpenAccount(rcptAcc.accountName) + acc, err := store.OpenAccount(log, rcptAcc.accountName) if err != nil { - log.Errorx("open account", err, mlog.Field("account", rcptAcc.accountName)) + log.Errorx("open account", err, slog.Any("account", rcptAcc.accountName)) metricDelivery.WithLabelValues("accounterror", "").Inc() addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing") continue @@ -2347,7 +2350,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW err = acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) { now := time.Now() defer func() { - log.Debugx("checking message and size delivery rates", retErr, mlog.Field("duration", time.Since(now))) + log.Debugx("checking message and size delivery rates", retErr, slog.Duration("duration", time.Since(now))) }() checkCount := func(msg store.Message, window time.Duration, limit int) { @@ -2440,7 +2443,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW RcptToDomain: rcptAcc.rcptTo.IPDomain.Domain.Name(), MsgFromLocalpart: msgFrom.Localpart, MsgFromDomain: msgFrom.Domain.Name(), - MsgFromOrgDomain: publicsuffix.Lookup(ctx, msgFrom.Domain).Name(), + MsgFromOrgDomain: publicsuffix.Lookup(ctx, log.Logger, msgFrom.Domain).Name(), EHLOValidated: ehloValidation == store.ValidationPass, MailFromValidated: mailFromValidation == store.ValidationPass, MsgFromValidated: msgFromValidation == store.ValidationStrict || msgFromValidation == store.ValidationDMARC || msgFromValidation == store.ValidationRelaxed, @@ -2616,7 +2619,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // We'll include all signatures for the organizational domain, even if they weren't // relevant due to strict alignment requirement. for _, dkimResult := range dkimResults { - if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, msgFrom.Domain) != publicsuffix.Lookup(ctx, dkimResult.Sig.Domain) { + if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, log.Logger, msgFrom.Domain) != publicsuffix.Lookup(ctx, log.Logger, dkimResult.Sig.Domain) { continue } r := dmarcrpt.DKIMAuthResult{ @@ -2684,7 +2687,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } } - log.Info("incoming message rejected", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom)) + log.Info("incoming message rejected", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom)) metricDelivery.WithLabelValues("reject", a.reason).Inc() c.setSlow(true) addError(rcptAcc, a.code, a.secode, a.userError, a.errmsg) @@ -2704,7 +2707,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } if a.tlsReport != nil { // todo future: add rate limiting to prevent DoS attacks. - if err := tlsrptdb.AddReport(ctx, msgFrom.Domain, c.mailFrom.String(), rcptAcc.destination.HostTLSReports, a.tlsReport); err != nil { + if err := tlsrptdb.AddReport(ctx, c.log, msgFrom.Domain, c.mailFrom.String(), rcptAcc.destination.HostTLSReports, a.tlsReport); err != nil { log.Errorx("saving TLSRPT report in database", err) } else { log.Info("tlsrpt report processed") @@ -2717,13 +2720,13 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // delivering. If this turns out to be a spammer, we've kept one of their // connections busy. if delayFirstTime && !m.IsForward && a.reason == reasonNoBadSignals && c.firstTimeSenderDelay > 0 { - log.Debug("delaying before delivering from sender without reputation", mlog.Field("delay", c.firstTimeSenderDelay)) + log.Debug("delaying before delivering from sender without reputation", slog.Duration("delay", c.firstTimeSenderDelay)) mox.Sleep(mox.Context, c.firstTimeSenderDelay) } // Gather the message-id before we deliver and the file may be consumed. if !parsedMessageID { - if p, err := message.Parse(c.log, false, store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil { + if p, err := message.Parse(c.log.Logger, false, store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil { log.Infox("parsing message for message-id", err) } else if header, err := p.Header(); err != nil { log.Infox("parsing message header for message-id", err) @@ -2735,11 +2738,11 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW if Localserve { code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart) if timeout { - c.log.Info("timing out due to special localpart") + log.Info("timing out due to special localpart") mox.Sleep(mox.Context, time.Hour) xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeOther00}, "timing out delivery due to special localpart") } else if code != 0 { - c.log.Info("failure due to special localpart", mlog.Field("code", code)) + log.Info("failure due to special localpart", slog.Int("code", code)) metricDelivery.WithLabelValues("delivererror", "localserve").Inc() addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code)) } @@ -2752,12 +2755,12 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW return } metricDelivery.WithLabelValues("delivered", a.reason).Inc() - log.Info("incoming message delivered", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom)) + log.Info("incoming message delivered", slog.String("reason", a.reason), slog.Any("msgfrom", msgFrom)) conf, _ := acc.Conf() if conf.RejectsMailbox != "" && m.MessageID != "" { if err := acc.RejectsRemove(log, conf.RejectsMailbox, m.MessageID); err != nil { - log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID)) + log.Errorx("removing message from rejects mailbox", err, slog.String("messageid", messageID)) } } }) diff --git a/smtpserver/server_test.go b/smtpserver/server_test.go index b092f78..5ec59fd 100644 --- a/smtpserver/server_test.go +++ b/smtpserver/server_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/config" @@ -106,15 +108,16 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test dmarcdb.EvalDB = nil } + log := mlog.New("smtpserver", nil) mox.Context = ctxbg mox.ConfigStaticPath = configPath mox.MustLoadConfig(true, false) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) os.RemoveAll(dataDir) var err error - ts.acc, err = store.OpenAccount("mjl") + ts.acc, err = store.OpenAccount(log, "mjl") tcheck(t, err, "open account") - err = ts.acc.SetPassword("testtest") + err = ts.acc.SetPassword(log, "testtest") tcheck(t, err, "set password") ts.switchStop = store.Switchboard() err = queue.Init() @@ -169,7 +172,8 @@ func (ts *testserver) run(fn func(helloErr error, client *smtpclient.Client)) { Auth: auth, RootCAs: mox.Conf.Static.TLS.CertPool, } - client, err := smtpclient.New(ctxbg, xlog.WithCid(ts.cid-1), clientConn, ts.tlsmode, ts.tlspkix, ourHostname, remoteHostname, opts) + log := pkglog.WithCid(ts.cid - 1) + client, err := smtpclient.New(ctxbg, log.Logger, clientConn, ts.tlsmode, ts.tlspkix, ourHostname, remoteHostname, opts) if err != nil { clientConn.Close() } else { @@ -355,13 +359,13 @@ func TestDelivery(t *testing.T) { } func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) { - mf, err := store.CreateMessageTemp("queue-dsn") + mf, err := store.CreateMessageTemp(pkglog, "queue-dsn") tcheck(t, err, "temp message") defer os.Remove(mf.Name()) defer mf.Close() _, err = mf.Write([]byte(msg)) tcheck(t, err, "write message") - err = acc.DeliverMailbox(xlog, mailbox, m, mf) + err = acc.DeliverMailbox(pkglog, mailbox, m, mf) tcheck(t, err, "deliver message") err = mf.Close() tcheck(t, err, "close message") @@ -376,7 +380,7 @@ func tretrain(t *testing.T, acc *store.Account) { bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom") os.Remove(dbPath) os.Remove(bloomPath) - jf, _, err := acc.OpenJunkFilter(ctxbg, xlog) + jf, _, err := acc.OpenJunkFilter(ctxbg, pkglog) tcheck(t, err, "open junk filter") defer jf.Close() @@ -1004,7 +1008,7 @@ func TestTLSReport(t *testing.T) { tcheck(t, xerr, "write msg") msg := msgb.String() - headers, xerr := dkim.Sign(ctxbg, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg)) + headers, xerr := dkim.Sign(ctxbg, pkglog.Logger, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg)) tcheck(t, xerr, "dkim sign") msg = headers + msg @@ -1040,7 +1044,7 @@ func TestRatelimitConnectionrate(t *testing.T) { // We'll be creating 300 connections, no TLS and reduce noise. ts.tlsmode = smtpclient.TLSSkip - mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelInfo}) + mlog.SetConfig(map[string]slog.Level{"": mlog.LevelInfo}) // We may be passing a window boundary during this tests. The limit is 300/minute. // So make twice that many connections and hope the tests don't take too long. @@ -1272,7 +1276,7 @@ func TestCatchall(t *testing.T) { tcheck(t, err, "checking delivered messages") tcompare(t, n, 3) - acc, err := store.OpenAccount("catchall") + acc, err := store.OpenAccount(pkglog, "catchall") tcheck(t, err, "open account") defer acc.Close() n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count() @@ -1361,7 +1365,7 @@ test email f, err := queue.OpenMessage(ctxbg, msgs[0].ID) tcheck(t, err, "open message in queue") defer f.Close() - results, err := dkim.Verify(ctxbg, resolver, false, dkim.DefaultPolicy, f, false) + results, err := dkim.Verify(ctxbg, pkglog.Logger, resolver, false, dkim.DefaultPolicy, f, false) tcheck(t, err, "verifying dkim message") tcompare(t, len(results), 1) tcompare(t, results[0].Status, dkim.StatusPass) @@ -1504,7 +1508,7 @@ test email tcheck(t, err, "listing queue") tcompare(t, len(msgs), 1) tcompare(t, msgs[0].RequireTLS, expRequireTLS) - _, err = queue.Drop(ctxbg, msgs[0].ID, "", "") + _, err = queue.Drop(ctxbg, pkglog, msgs[0].ID, "", "") tcheck(t, err, "deleting message from queue") }) } diff --git a/spf/spf.go b/spf/spf.go index 3619edf..8f39476 100644 --- a/spf/spf.go +++ b/spf/spf.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -28,8 +30,6 @@ import ( // sure we make names absolute when looking up. For verifying, we do not want to // verify names relative to our local search domain. -var xlog = mlog.New("spf") - var ( metricSPFVerify = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -129,11 +129,11 @@ var timeNow = time.Now // Lookup looks up and parses an SPF TXT record for domain. // // authentic indicates if the DNS results were DNSSEC-verified. -func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rstatus Status, rtxt string, rrecord *Record, authentic bool, rerr error) { - log := xlog.WithContext(ctx) +func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rstatus Status, rtxt string, rrecord *Record, authentic bool, rerr error) { + log := mlog.New("spf", elog) start := time.Now() defer func() { - log.Debugx("spf lookup result", rerr, mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) + log.Debugx("spf lookup result", rerr, slog.Any("domain", domain), slog.Any("status", rstatus), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start))) }() // ../rfc/7208:586 @@ -194,12 +194,12 @@ func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rsta // of 2 lookups resulting in no records ("void lookups"). // // authentic indicates if the DNS results were DNSSEC-verified. -func Verify(ctx context.Context, resolver dns.Resolver, args Args) (received Received, domain dns.Domain, explanation string, authentic bool, rerr error) { - log := xlog.WithContext(ctx) +func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, args Args) (received Received, domain dns.Domain, explanation string, authentic bool, rerr error) { + log := mlog.New("spf", elog) start := time.Now() defer func() { metricSPFVerify.WithLabelValues(string(received.Result)).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("spf verify result", rerr, mlog.Field("domain", args.domain), mlog.Field("ip", args.RemoteIP), mlog.Field("status", received.Result), mlog.Field("explanation", explanation), mlog.Field("duration", time.Since(start))) + log.Debugx("spf verify result", rerr, slog.Any("domain", args.domain), slog.Any("ip", args.RemoteIP), slog.Any("status", received.Result), slog.String("explanation", explanation), slog.Duration("duration", time.Since(start))) }() isHello, ok := prepare(&args) @@ -215,7 +215,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, args Args) (received Rec return received, dns.Domain{}, "", false, nil } - status, mechanism, expl, authentic, err := checkHost(ctx, resolver, args) + status, mechanism, expl, authentic, err := checkHost(ctx, log, resolver, args) comment := fmt.Sprintf("domain %s", args.domain.ASCII) if isHello { comment += ", from ehlo because mailfrom is empty" @@ -272,37 +272,35 @@ func prepare(args *Args) (isHello bool, ok bool) { } // lookup spf record, then evaluate args against it. -func checkHost(ctx context.Context, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { - status, _, record, rauthentic, err := Lookup(ctx, resolver, args.domain) +func checkHost(ctx context.Context, log mlog.Log, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { + status, _, record, rauthentic, err := Lookup(ctx, log.Logger, resolver, args.domain) if err != nil { return status, "", "", rauthentic, err } var evalAuthentic bool - rstatus, mechanism, rexplanation, evalAuthentic, rerr = evaluate(ctx, record, resolver, args) + rstatus, mechanism, rexplanation, evalAuthentic, rerr = evaluate(ctx, log, record, resolver, args) rauthentic = rauthentic && evalAuthentic return } // Evaluate evaluates the IP and names from args against the SPF DNS record for the domain. -func Evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { +func Evaluate(ctx context.Context, elog *slog.Logger, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { + log := mlog.New("spf", elog) _, ok := prepare(&args) if !ok { return StatusNone, "default", "", false, fmt.Errorf("no domain name to validate") } - return evaluate(ctx, record, resolver, args) + return evaluate(ctx, log, record, resolver, args) } // evaluate RemoteIP against domain from args, given record. -func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { - log := xlog.WithContext(ctx) +func evaluate(ctx context.Context, log mlog.Log, record *Record, resolver dns.Resolver, args Args) (rstatus Status, mechanism, rexplanation string, rauthentic bool, rerr error) { start := time.Now() defer func() { - log.Debugx("spf evaluate result", rerr, mlog.Field("dnsrequests", *args.dnsRequests), mlog.Field("voidlookups", *args.voidLookups), mlog.Field("domain", args.domain), mlog.Field("status", rstatus), mlog.Field("mechanism", mechanism), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start))) + log.Debugx("spf evaluate result", rerr, slog.Int("dnsrequests", *args.dnsRequests), slog.Int("voidlookups", *args.voidLookups), slog.Any("domain", args.domain), slog.Any("status", rstatus), slog.String("mechanism", mechanism), slog.String("explanation", rexplanation), slog.Duration("duration", time.Since(start))) }() - resolver = dns.WithPackage(resolver, "spf") - if args.dnsRequests == nil { args.dnsRequests = new(int) args.voidLookups = new(int) @@ -388,7 +386,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A nargs := args nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")} nargs.explanation = &record.Explanation // ../rfc/7208:1548 - status, _, _, authentic, err := checkHost(ctx, resolver, nargs) + status, _, _, authentic, err := checkHost(ctx, log, resolver, nargs) rauthentic = rauthentic && authentic // ../rfc/7208:1202 switch status { @@ -477,7 +475,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A for _, rname := range rnames { rd, err := dns.ParseDomain(strings.TrimSuffix(rname, ".")) if err != nil { - log.Errorx("bad address in ptr record", err, mlog.Field("address", rname)) + log.Errorx("bad address in ptr record", err, slog.String("address", rname)) continue } // ../rfc/7208-eid4751 ../rfc/7208:1323 @@ -565,7 +563,7 @@ func evaluate(ctx context.Context, record *Record, resolver dns.Resolver, args A nargs := args nargs.domain = dns.Domain{ASCII: strings.TrimSuffix(name, ".")} nargs.explanation = nil // ../rfc/7208:1548 - status, mechanism, expl, authentic, err := checkHost(ctx, resolver, nargs) + status, mechanism, expl, authentic, err := checkHost(ctx, log, resolver, nargs) rauthentic = rauthentic && authentic if status == StatusNone { return StatusPermerror, mechanism, "", rauthentic, err diff --git a/spf/spf_test.go b/spf/spf_test.go index 2b320ec..3506964 100644 --- a/spf/spf_test.go +++ b/spf/spf_test.go @@ -10,9 +10,12 @@ import ( "time" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/smtp" ) +var pkglog = mlog.New("spf", nil) + func TestLookup(t *testing.T) { resolver := dns.MockResolver{ TXT: map[string][]string{ @@ -31,7 +34,7 @@ func TestLookup(t *testing.T) { t.Helper() d := dns.Domain{ASCII: domain} - status, txt, record, _, err := Lookup(context.Background(), resolver, d) + status, txt, record, _, err := Lookup(context.Background(), pkglog.Logger, resolver, d) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("got err %v, expected err %v", err, expErr) } @@ -259,7 +262,7 @@ func TestVerify(t *testing.T) { LocalIP: xip("127.0.0.1"), LocalHostname: dns.Domain{ASCII: "localhost"}, } - received, _, _, _, err := Verify(ctx, r, args) + received, _, _, _, err := Verify(ctx, pkglog.Logger, r, args) if received.Result != status { t.Fatalf("got status %q, expected %q, for ip %q (err %v)", received.Result, status, ip, err) } @@ -345,7 +348,7 @@ func TestVerifyMultipleDomain(t *testing.T) { LocalIP: net.ParseIP("127.0.0.1"), LocalHostname: dns.Domain{ASCII: "localhost"}, } - received, _, _, _, err := Verify(context.Background(), resolver, args) + received, _, _, _, err := Verify(context.Background(), pkglog.Logger, resolver, args) if err != nil { t.Fatalf("unexpected error: %s", err) } @@ -370,7 +373,7 @@ func TestVerifyScenarios(t *testing.T) { test := func(resolver dns.Resolver, args Args, expStatus Status, expDomain string, expExpl string, expErr error) { t.Helper() - recv, d, expl, _, err := Verify(context.Background(), resolver, args) + recv, d, expl, _, err := Verify(context.Background(), pkglog.Logger, resolver, args) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("got err %v, expected %v", err, expErr) } @@ -505,7 +508,7 @@ func TestEvaluate(t *testing.T) { record := &Record{} resolver := dns.MockResolver{} args := Args{} - status, _, _, _, _ := Evaluate(context.Background(), record, resolver, args) + status, _, _, _, _ := Evaluate(context.Background(), pkglog.Logger, record, resolver, args) if status != StatusNone { t.Fatalf("got status %q, expected none", status) } @@ -513,7 +516,7 @@ func TestEvaluate(t *testing.T) { args = Args{ HelloDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "test.example"}}, } - status, mechanism, _, _, err := Evaluate(context.Background(), record, resolver, args) + status, mechanism, _, _, err := Evaluate(context.Background(), pkglog.Logger, record, resolver, args) if status != StatusNeutral || mechanism != "default" || err != nil { t.Fatalf("got status %q, mechanism %q, err %v, expected neutral, default, no error", status, mechanism, err) } diff --git a/store/account.go b/store/account.go index 69d72ed..c06b2eb 100644 --- a/store/account.go +++ b/store/account.go @@ -44,6 +44,7 @@ import ( "golang.org/x/crypto/bcrypt" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "golang.org/x/text/unicode/norm" "github.com/mjl-/bstore" @@ -67,8 +68,6 @@ import ( // false again. var CheckConsistencyOnClose = true -var xlog = mlog.New("store") - var ( ErrUnknownMailbox = errors.New("no such mailbox") ErrUnknownCredentials = errors.New("credentials not found") @@ -577,15 +576,15 @@ func (m *Message) PrepareExpunge() { // PrepareThreading sets MessageID and SubjectBase (used in threading) based on the // envelope in part. -func (m *Message) PrepareThreading(log *mlog.Log, part *message.Part) { +func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) { if part.Envelope == nil { return } messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID) if err != nil { - log.Debugx("parsing message-id, ignoring", err, mlog.Field("messageid", part.Envelope.MessageID)) + log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID)) } else if raw { - log.Debug("could not parse message-id as address, continuing with raw value", mlog.Field("messageid", part.Envelope.MessageID)) + log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID)) } m.MessageID = messageID m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false) @@ -747,7 +746,7 @@ func closeAccount(acc *Account) (rerr error) { // // No additional data path prefix or ".db" suffix should be added to the name. // A single shared account exists per name. -func OpenAccount(name string) (*Account, error) { +func OpenAccount(log mlog.Log, name string) (*Account, error) { openAccounts.Lock() defer openAccounts.Unlock() if acc, ok := openAccounts.names[name]; ok { @@ -759,7 +758,7 @@ func OpenAccount(name string) (*Account, error) { return nil, ErrAccountUnknown } - acc, err := openAccount(name) + acc, err := openAccount(log, name) if err != nil { return nil, err } @@ -768,15 +767,15 @@ func OpenAccount(name string) (*Account, error) { } // openAccount opens an existing account, or creates it if it is missing. -func openAccount(name string) (a *Account, rerr error) { +func openAccount(log mlog.Log, name string) (a *Account, rerr error) { dir := filepath.Join(mox.DataDirPath("accounts"), name) - return OpenAccountDB(dir, name) + return OpenAccountDB(log, dir, name) } // OpenAccountDB opens an account database file and returns an initialized account // or error. Only exported for use by subcommands that verify the database file. // Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth. -func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { +func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) { dbpath := filepath.Join(accountDir, "index.db") // Create account if it doesn't exist yet. @@ -823,7 +822,7 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error { if !mentioned { mentioned = true - xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName)) + log.Info("first calculation of mailbox counts for account", slog.String("account", accountName)) } mc, err := mb.CalculateCounts(tx) if err != nil { @@ -866,17 +865,17 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { // Ensure all messages have a MessageID and SubjectBase, which are needed when // matching threads. // Then assign messages to threads, in the same way we do during imports. - xlog.Info("upgrading account for threading, in background", mlog.Field("account", acc.Name)) + log.Info("upgrading account for threading, in background", slog.String("account", acc.Name)) go func() { defer func() { err := closeAccount(acc) - xlog.Check(err, "closing use of account after upgrading account storage for threads", mlog.Field("account", a.Name)) + log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name)) }() defer func() { x := recover() // Should not happen, but don't take program down if it does. if x != nil { - xlog.Error("upgradeThreads panic", mlog.Field("err", x)) + log.Error("upgradeThreads panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Upgradethreads) acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x) @@ -886,12 +885,12 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { close(acc.threadsCompleted) }() - err := upgradeThreads(mox.Shutdown, acc, &up) + err := upgradeThreads(mox.Shutdown, log, acc, &up) if err != nil { a.threadsErr = err - xlog.Errorx("upgrading account for threading, aborted", err, mlog.Field("account", a.Name)) + log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name)) } else { - xlog.Info("upgrading account for threading, completed", mlog.Field("account", a.Name)) + log.Info("upgrading account for threading, completed", slog.String("account", a.Name)) } }() return acc, nil @@ -901,7 +900,7 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { // account has completed, and returns an error if not successful. // // To be used before starting an import of messages. -func (a *Account) ThreadingWait(log *mlog.Log) error { +func (a *Account) ThreadingWait(log mlog.Log) error { select { case <-a.threadsCompleted: return a.threadsErr @@ -1224,7 +1223,7 @@ func (a *Account) WithRLock(fn func()) { // Caller must broadcast new message. // // Caller must update mailbox counts. -func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error { +func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads bool) error { if m.Expunged { return fmt.Errorf("cannot deliver expunged message") } @@ -1245,9 +1244,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile. var part *message.Part if m.ParsedBuf == nil { - p, err := message.EnsurePart(log, false, mr, m.Size) + p, err := message.EnsurePart(log.Logger, false, mr, m.Size) if err != nil { - log.Infox("parsing delivered message", err, mlog.Field("parse", ""), mlog.Field("message", m.ID)) + log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID)) // We continue, p is still valid. } part = &p @@ -1259,7 +1258,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi } else { var p message.Part if err := json.Unmarshal(m.ParsedBuf, &p); err != nil { - log.Errorx("unmarshal parsed message, continuing", err, mlog.Field("parse", "")) + log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", "")) } else { part = &p } @@ -1328,19 +1327,19 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi for _, addr := range addrs { if addr.User == "" { // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero. - log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", mlog.Field("address", addr)) + log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr)) continue } d, err := dns.ParseDomain(addr.Host) if err != nil { - log.Debugx("parsing domain in to/cc/bcc address", err, mlog.Field("address", addr)) + log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr)) continue } mr := Recipient{ MessageID: m.ID, Localpart: smtp.Localpart(addr.User), Domain: d.Name(), - OrgDomain: publicsuffix.Lookup(context.TODO(), d).Name(), + OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(), Sent: sent, } if err := tx.Insert(&mr); err != nil { @@ -1365,9 +1364,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi } if sync { - if err := moxio.SyncDir(msgDir); err != nil { + if err := moxio.SyncDir(log, msgDir); err != nil { xerr := os.Remove(msgPath) - log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath)) + log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath)) return fmt.Errorf("sync directory: %w", err) } } @@ -1376,7 +1375,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi l := []Message{*m} if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil { xerr := os.Remove(msgPath) - log.Check(xerr, "removing message after syncdir error", mlog.Field("path", msgPath)) + log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath)) return fmt.Errorf("training junkfilter: %w", err) } *m = l[0] @@ -1387,7 +1386,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi // SetPassword saves a new password for this account. This password is used for // IMAP, SMTP (submission) sessions and the HTTP account web page. -func (a *Account) SetPassword(password string) error { +func (a *Account) SetPassword(log mlog.Log, password string) error { hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { return fmt.Errorf("generating password hash: %w", err) @@ -1439,7 +1438,7 @@ func (a *Account) SetPassword(password string) error { return nil }) if err == nil { - xlog.Info("new password set for account", mlog.Field("account", a.Name)) + log.Info("new password set for account", slog.String("account", a.Name)) } return err } @@ -1590,21 +1589,21 @@ func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, erro // MessageRuleset returns the first ruleset (if any) that message the message // represented by msgPrefix and msgFile, with smtp and validation fields from m. -func MessageRuleset(log *mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset { +func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset { if len(dest.Rulesets) == 0 { return nil } mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile. - p, err := message.Parse(log, false, mr) + p, err := message.Parse(log.Logger, false, mr) if err != nil { - log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, mlog.Field("parse", "")) + log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", "")) // note: part is still set. } // todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing. header, err := p.Header() if err != nil { - log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, mlog.Field("parse", "")) + log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", "")) // todo: reject message? return nil } @@ -1678,7 +1677,7 @@ func (a *Account) MessageReader(m Message) *MsgReader { // Caller must hold account wlock (mailbox may be created). // Message delivery, possible mailbox creation, and updated mailbox counts are // broadcasted. -func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error { +func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error { var mailbox string rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile) if rs != nil { @@ -1696,7 +1695,7 @@ func (a *Account) DeliverDestination(log *mlog.Log, dest config.Destination, m * // Caller must hold account wlock (mailbox may be created). // Message delivery, possible mailbox creation, and updated mailbox counts are // broadcasted. -func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgFile *os.File) error { +func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error { var changes []Change err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error { mb, chl, err := a.MailboxEnsure(tx, mailbox, true) @@ -1734,7 +1733,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF // // Caller most hold account wlock. // Changes are broadcasted. -func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) { +func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) { var changes []Change var remove []Message @@ -1742,7 +1741,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS for _, m := range remove { p := a.MessagePath(m.ID) err := os.Remove(p) - log.Check(err, "removing rejects message file", mlog.Field("path", p)) + log.Check(err, "removing rejects message file", slog.String("path", p)) } }() @@ -1796,7 +1795,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS return hasSpace, nil } -func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) { +func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) { if len(l) == 0 { return nil, nil } @@ -1858,7 +1857,7 @@ func (a *Account) rejectsRemoveMessages(ctx context.Context, log *mlog.Log, tx * // RejectsRemove removes a message from the rejects mailbox if present. // Caller most hold account wlock. // Changes are broadcasted. -func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string) error { +func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error { var changes []Change var remove []Message @@ -1866,7 +1865,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string) for _, m := range remove { p := a.MessagePath(m.ID) err := os.Remove(p) - log.Check(err, "removing rejects message file", mlog.Field("path", p)) + log.Check(err, "removing rejects message file", slog.String("path", p)) } }() @@ -1933,8 +1932,8 @@ func manageAuthCache() { // OpenEmailAuth opens an account given an email address and password. // // The email address may contain a catchall separator. -func OpenEmailAuth(email string, password string) (acc *Account, rerr error) { - acc, _, rerr = OpenEmail(email) +func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) { + acc, _, rerr = OpenEmail(log, email) if rerr != nil { return } @@ -1942,7 +1941,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) { defer func() { if rerr != nil && acc != nil { err := acc.Close() - xlog.Check(err, "closing account after open auth failure") + log.Check(err, "closing account after open auth failure") acc = nil } }() @@ -1973,7 +1972,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) { // OpenEmail opens an account given an email address. // // The email address may contain a catchall separator. -func OpenEmail(email string) (*Account, config.Destination, error) { +func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) { addr, err := smtp.ParseAddress(email) if err != nil { return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err) @@ -1984,7 +1983,7 @@ func OpenEmail(email string) (*Account, config.Destination, error) { } else if err != nil { return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err) } - acc, err := OpenAccount(accountName) + acc, err := OpenAccount(log, accountName) if err != nil { return nil, config.Destination{}, err } @@ -2383,7 +2382,7 @@ func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (chang // indicates that and an error is returned. // // Caller should broadcast the changes and remove files for the removed message IDs. -func (a *Account) MailboxDelete(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) { +func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) { // Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about // NoInferior and NoSelect. We just require only leaf mailboxes are deleted. qmb := bstore.QueryTx[Mailbox](tx) diff --git a/store/account_test.go b/store/account_test.go index 9eee51c..de21dbe 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -19,6 +19,7 @@ import ( ) var ctxbg = context.Background() +var pkglog = mlog.New("store", nil) func tcheck(t *testing.T, err error, msg string) { t.Helper() @@ -28,10 +29,11 @@ func tcheck(t *testing.T, err error, msg string) { } func TestMailbox(t *testing.T) { + log := mlog.New("store", nil) os.RemoveAll("../testdata/store/data") mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf") mox.MustLoadConfig(true, false) - acc, err := OpenAccount("mjl") + acc, err := OpenAccount(log, "mjl") tcheck(t, err, "open account") defer func() { err = acc.Close() @@ -39,9 +41,7 @@ func TestMailbox(t *testing.T) { }() defer Switchboard()() - log := mlog.New("store") - - msgFile, err := CreateMessageTemp("account-test") + msgFile, err := CreateMessageTemp(log, "account-test") if err != nil { t.Fatalf("creating temp msg file: %s", err) } @@ -72,7 +72,7 @@ func TestMailbox(t *testing.T) { } acc.WithWLock(func() { conf, _ := acc.Conf() - err := acc.DeliverDestination(xlog, conf.Destinations["mjl"], &m, msgFile) + err := acc.DeliverDestination(log, conf.Destinations["mjl"], &m, msgFile) tcheck(t, err, "deliver without consume") err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error { @@ -81,7 +81,7 @@ func TestMailbox(t *testing.T) { tcheck(t, err, "sent mailbox") msent.MailboxID = mbsent.ID msent.MailboxOrigID = mbsent.ID - err = acc.DeliverMessage(xlog, tx, &msent, msgFile, true, false, false) + err = acc.DeliverMessage(pkglog, tx, &msent, msgFile, true, false, false) tcheck(t, err, "deliver message") if !msent.ThreadMuted || !msent.ThreadCollapsed { t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m") @@ -97,7 +97,7 @@ func TestMailbox(t *testing.T) { tcheck(t, err, "insert rejects mailbox") mreject.MailboxID = mbrejects.ID mreject.MailboxOrigID = mbrejects.ID - err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, true, false, false) + err = acc.DeliverMessage(pkglog, tx, &mreject, msgFile, true, false, false) tcheck(t, err, "deliver message") err = tx.Get(&mbrejects) @@ -110,7 +110,7 @@ func TestMailbox(t *testing.T) { }) tcheck(t, err, "deliver as sent and rejects") - err = acc.DeliverDestination(xlog, conf.Destinations["mjl"], &mconsumed, msgFile) + err = acc.DeliverDestination(pkglog, conf.Destinations["mjl"], &mconsumed, msgFile) tcheck(t, err, "deliver with consume") err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error { @@ -141,7 +141,7 @@ func TestMailbox(t *testing.T) { }) tcheck(t, err, "untraining non-junk") - err = acc.SetPassword("testtest") + err = acc.SetPassword(log, "testtest") tcheck(t, err, "set password") key0, err := acc.Subjectpass("test@localhost") @@ -223,37 +223,37 @@ func TestMailbox(t *testing.T) { // Run the auth tests twice for possible cache effects. for i := 0; i < 2; i++ { - _, err := OpenEmailAuth("mjl@mox.example", "bogus") + _, err := OpenEmailAuth(log, "mjl@mox.example", "bogus") if err != ErrUnknownCredentials { t.Fatalf("got %v, expected ErrUnknownCredentials", err) } } for i := 0; i < 2; i++ { - acc2, err := OpenEmailAuth("mjl@mox.example", "testtest") + acc2, err := OpenEmailAuth(log, "mjl@mox.example", "testtest") tcheck(t, err, "open for email with auth") err = acc2.Close() tcheck(t, err, "close account") } - acc2, err := OpenEmailAuth("other@mox.example", "testtest") + acc2, err := OpenEmailAuth(log, "other@mox.example", "testtest") tcheck(t, err, "open for email with auth") err = acc2.Close() tcheck(t, err, "close account") - _, err = OpenEmailAuth("bogus@mox.example", "testtest") + _, err = OpenEmailAuth(log, "bogus@mox.example", "testtest") if err != ErrUnknownCredentials { t.Fatalf("got %v, expected ErrUnknownCredentials", err) } - _, err = OpenEmailAuth("mjl@test.example", "testtest") + _, err = OpenEmailAuth(log, "mjl@test.example", "testtest") if err != ErrUnknownCredentials { t.Fatalf("got %v, expected ErrUnknownCredentials", err) } } func TestMessageRuleset(t *testing.T) { - f, err := CreateMessageTemp("msgruleset") + f, err := CreateMessageTemp(pkglog, "msgruleset") tcheck(t, err, "creating temp msg file") defer os.Remove(f.Name()) defer f.Close() @@ -284,7 +284,7 @@ Rulesets: } dest.Rulesets[0].HeadersRegexpCompiled = hdrs - c := MessageRuleset(xlog, dest, &Message{}, msgBuf, f) + c := MessageRuleset(pkglog, dest, &Message{}, msgBuf, f) if c == nil { t.Fatalf("expected ruleset match") } @@ -293,7 +293,7 @@ Rulesets: test `, "\n", "\r\n")) - c = MessageRuleset(xlog, dest, &Message{}, msg2Buf, f) + c = MessageRuleset(pkglog, dest, &Message{}, msg2Buf, f) if c != nil { t.Fatalf("expected no ruleset match") } diff --git a/store/cleanuptemp.go b/store/cleanuptemp.go index 0bfc757..10bb148 100644 --- a/store/cleanuptemp.go +++ b/store/cleanuptemp.go @@ -3,15 +3,17 @@ package store import ( "os" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/mlog" ) // CloseRemoveTempFile closes and removes f, a file described by descr. Often // used in a defer after creating a temporary file. -func CloseRemoveTempFile(log *mlog.Log, f *os.File, descr string) { +func CloseRemoveTempFile(log mlog.Log, f *os.File, descr string) { name := f.Name() err := f.Close() - log.Check(err, "closing temporary file", mlog.Field("kind", descr)) + log.Check(err, "closing temporary file", slog.String("kind", descr)) err = os.Remove(name) - log.Check(err, "removing temporary file", mlog.Field("kind", descr)) + log.Check(err, "removing temporary file", slog.String("kind", descr)) } diff --git a/store/export.go b/store/export.go index fe4f15f..527b05c 100644 --- a/store/export.go +++ b/store/export.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/mlog" @@ -106,7 +108,7 @@ func (a DirArchiver) Close() error { // Some errors are not fatal and result in skipped messages. In that happens, a // file "errors.txt" is added to the archive describing the errors. The goal is to // let users export (hopefully) most messages even in the face of errors. -func ExportMessages(ctx context.Context, log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error { +func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error { // todo optimize: should prepare next file to add to archive (can be an mbox with many messages) while writing a file to the archive (which typically compresses, which takes time). // Start transaction without closure, we are going to close it early, but don't @@ -291,7 +293,7 @@ func ExportMessages(ctx context.Context, log *mlog.Log, db *bstore.DB, accountDi err = mboxtmp.Close() log.Check(err, "closing temporary mbox file") err = os.Remove(name) - log.Check(err, "removing temporary mbox file", mlog.Field("path", name)) + log.Check(err, "removing temporary mbox file", slog.String("path", name)) mboxwriter = nil mboxtmp = nil return nil diff --git a/store/export_test.go b/store/export_test.go index 8cdb928..e2abad5 100644 --- a/store/export_test.go +++ b/store/export_test.go @@ -22,14 +22,14 @@ func TestExport(t *testing.T) { os.RemoveAll("../testdata/store/data") mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf") mox.MustLoadConfig(true, false) - acc, err := OpenAccount("mjl") + acc, err := OpenAccount(pkglog, "mjl") tcheck(t, err, "open account") defer acc.Close() defer Switchboard()() - log := mlog.New("export") + log := mlog.New("export", nil) - msgFile, err := CreateMessageTemp("mox-test-export") + msgFile, err := CreateMessageTemp(pkglog, "mox-test-export") tcheck(t, err, "create temp") defer os.Remove(msgFile.Name()) // To be sure. defer msgFile.Close() @@ -38,11 +38,11 @@ func TestExport(t *testing.T) { tcheck(t, err, "write message") m := Message{Received: time.Now(), Size: int64(len(msg))} - err = acc.DeliverMailbox(xlog, "Inbox", &m, msgFile) + err = acc.DeliverMailbox(pkglog, "Inbox", &m, msgFile) tcheck(t, err, "deliver") m = Message{Received: time.Now(), Size: int64(len(msg))} - err = acc.DeliverMailbox(xlog, "Trash", &m, msgFile) + err = acc.DeliverMailbox(pkglog, "Trash", &m, msgFile) tcheck(t, err, "deliver") var maildirZip, maildirTar, mboxZip, mboxTar bytes.Buffer diff --git a/store/import.go b/store/import.go index dedc55e..e6528bb 100644 --- a/store/import.go +++ b/store/import.go @@ -13,6 +13,7 @@ import ( "time" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/mjl-/mox/mlog" ) @@ -25,25 +26,25 @@ type MsgSource interface { // MboxReader reads messages from an mbox file, implementing MsgSource. type MboxReader struct { - createTemp func(pattern string) (*os.File, error) + log mlog.Log + createTemp func(log mlog.Log, pattern string) (*os.File, error) path string line int r *bufio.Reader prevempty bool nonfirst bool - log *mlog.Log eof bool fromLine string // "From "-line for this message. header bool // Now in header section. } -func NewMboxReader(createTemp func(pattern string) (*os.File, error), filename string, r io.Reader, log *mlog.Log) *MboxReader { +func NewMboxReader(log mlog.Log, createTemp func(log mlog.Log, pattern string) (*os.File, error), filename string, r io.Reader) *MboxReader { return &MboxReader{ + log: log, createTemp: createTemp, path: filename, line: 1, r: bufio.NewReader(r), - log: log, } } @@ -78,7 +79,7 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) { mr.fromLine = strings.TrimSpace(string(line)) } - f, err := mr.createTemp("mboxreader") + f, err := mr.createTemp(mr.log, "mboxreader") if err != nil { return nil, nil, mr.Position(), err } @@ -202,22 +203,22 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) { } type MaildirReader struct { - createTemp func(pattern string) (*os.File, error) + log mlog.Log + createTemp func(log mlog.Log, pattern string) (*os.File, error) newf, curf *os.File f *os.File // File we are currently reading from. We first read newf, then curf. dir string // Name of directory for f. Can be empty on first call. entries []os.DirEntry dovecotFlags []string // Lower-case flags/keywords. - log *mlog.Log } -func NewMaildirReader(createTemp func(pattern string) (*os.File, error), newf, curf *os.File, log *mlog.Log) *MaildirReader { +func NewMaildirReader(log mlog.Log, createTemp func(log mlog.Log, pattern string) (*os.File, error), newf, curf *os.File) *MaildirReader { mr := &MaildirReader{ + log: log, createTemp: createTemp, newf: newf, curf: curf, f: newf, - log: log, } // Best-effort parsing of dovecot keywords. @@ -263,7 +264,7 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) { err := sf.Close() mr.log.Check(err, "closing message file after error") }() - f, err := mr.createTemp("maildirreader") + f, err := mr.createTemp(mr.log, "maildirreader") if err != nil { return nil, nil, p, err } @@ -273,7 +274,7 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) { err := f.Close() mr.log.Check(err, "closing temporary message file after maildir read error") err = os.Remove(name) - mr.log.Check(err, "removing temporary message file after maildir read error", mlog.Field("path", name)) + mr.log.Check(err, "removing temporary message file after maildir read error", slog.String("path", name)) } }() @@ -370,7 +371,7 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) { // returns valid flags/keywords, as lower-case. If an error is encountered and // returned, any keywords that were found are still returned. The returned list has // both system/well-known flags and custom keywords. -func ParseDovecotKeywordsFlags(r io.Reader, log *mlog.Log) ([]string, error) { +func ParseDovecotKeywordsFlags(r io.Reader, log mlog.Log) ([]string, error) { /* If the dovecot-keywords file is present, we parse its additional flags, see https://doc.dovecot.org/admin_manual/mailbox_formats/maildir/ diff --git a/store/import_test.go b/store/import_test.go index 233bfcc..0fa741d 100644 --- a/store/import_test.go +++ b/store/import_test.go @@ -10,7 +10,7 @@ import ( ) func TestMboxReader(t *testing.T) { - createTemp := func(pattern string) (*os.File, error) { + createTemp := func(log mlog.Log, pattern string) (*os.File, error) { return os.CreateTemp("", pattern) } mboxf, err := os.Open("../testdata/importtest.mbox") @@ -19,7 +19,8 @@ func TestMboxReader(t *testing.T) { } defer mboxf.Close() - mr := NewMboxReader(createTemp, mboxf.Name(), mboxf, mlog.New("mboxreader")) + log := mlog.New("mboxreader", nil) + mr := NewMboxReader(log, createTemp, mboxf.Name(), mboxf) _, mf0, _, err := mr.Next() if err != nil { t.Fatalf("next mbox message: %v", err) @@ -41,7 +42,7 @@ func TestMboxReader(t *testing.T) { } func TestMaildirReader(t *testing.T) { - createTemp := func(pattern string) (*os.File, error) { + createTemp := func(log mlog.Log, pattern string) (*os.File, error) { return os.CreateTemp("", pattern) } // todo: rename 1642966915.1.mox to "1642966915.1.mox:2,"? cannot have that name in the git repo because go module (or the proxy) doesn't like it. could also add some flags and test they survive the import. @@ -57,7 +58,8 @@ func TestMaildirReader(t *testing.T) { } defer curf.Close() - mr := NewMaildirReader(createTemp, newf, curf, mlog.New("maildirreader")) + log := mlog.New("maildirreader", nil) + mr := NewMaildirReader(log, createTemp, newf, curf) _, mf0, _, err := mr.Next() if err != nil { t.Fatalf("next maildir message: %v", err) @@ -85,7 +87,7 @@ func TestParseDovecotKeywords(t *testing.T) { 3 $Forwarded 4 $Junk ` - flags, err := ParseDovecotKeywordsFlags(strings.NewReader(data), mlog.New("dovecotkeywords")) + flags, err := ParseDovecotKeywordsFlags(strings.NewReader(data), mlog.New("dovecotkeywords", nil)) if err != nil { t.Fatalf("parsing dovecot-keywords: %v", err) } diff --git a/store/search.go b/store/search.go index 8a9f8fb..ec4858a 100644 --- a/store/search.go +++ b/store/search.go @@ -57,7 +57,7 @@ func PrepareWordSearch(words, notWords []string) WordSearch { // The search terms are matched against content-transfer-decoded and // charset-decoded bodies and optionally headers. // HTML parts are currently treated as regular text, without parsing HTML. -func (ws WordSearch) MatchPart(log *mlog.Log, p *message.Part, headerToo bool) (bool, error) { +func (ws WordSearch) MatchPart(log mlog.Log, p *message.Part, headerToo bool) (bool, error) { seen := map[int]bool{} miss, err := ws.matchPart(log, p, headerToo, seen) match := err == nil && !miss && len(seen) == len(ws.words) @@ -73,7 +73,7 @@ func (ws WordSearch) isQuickHit(seen map[int]bool) bool { // search a part as text and/or its subparts, recursively. Once we know we have // a miss, we stop (either due to not-word match or error). In case of // non-miss, the caller checks if there was a hit. -func (ws WordSearch) matchPart(log *mlog.Log, p *message.Part, headerToo bool, seen map[int]bool) (miss bool, rerr error) { +func (ws WordSearch) matchPart(log mlog.Log, p *message.Part, headerToo bool, seen map[int]bool) (miss bool, rerr error) { if headerToo { miss, err := ws.searchReader(log, p.HeaderReader(), seen) if miss || err != nil || ws.isQuickHit(seen) { @@ -108,7 +108,7 @@ func (ws WordSearch) matchPart(log *mlog.Log, p *message.Part, headerToo bool, s return false, nil } -func (ws WordSearch) searchReader(log *mlog.Log, r io.Reader, seen map[int]bool) (miss bool, rerr error) { +func (ws WordSearch) searchReader(log mlog.Log, r io.Reader, seen map[int]bool) (miss bool, rerr error) { // We will be reading through the content, stopping as soon as we known an answer: // when all words have been seen and there are no "not words" (true), or one "not // word" has been seen (false). We use bytes.Contains to look for the words. We diff --git a/store/threads.go b/store/threads.go index 80fb37c..5b9c82c 100644 --- a/store/threads.go +++ b/store/threads.go @@ -11,6 +11,7 @@ import ( "time" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/mjl-/bstore" @@ -26,7 +27,7 @@ import ( // may have a threadid 0. That results in this message getting threadid 0, which // will handled by the background upgrade process assigning a threadid when it gets // to this message. -func assignThread(log *mlog.Log, tx *bstore.Tx, m *Message, part *message.Part) error { +func assignThread(log mlog.Log, tx *bstore.Tx, m *Message, part *message.Part) error { if m.MessageID != "" { // Match against existing different message with same Message-ID. q := bstore.QueryTx[Message](tx) @@ -47,11 +48,11 @@ func assignThread(log *mlog.Log, tx *bstore.Tx, m *Message, part *message.Part) h, err := part.Header() if err != nil { - log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID)) + log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, slog.Int64("msgid", m.ID)) } messageIDs, err := message.ReferencedIDs(h.Values("References"), h.Values("In-Reply-To")) if err != nil { - log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID)) + log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, slog.Int64("msgid", m.ID)) } for i := len(messageIDs) - 1; i >= 0; i-- { messageID := messageIDs[i] @@ -123,7 +124,7 @@ func assignParent(m *Message, pm Message, updateSeen bool) { // // ModSeq is not changed. Calles should bump the uid validity of the mailboxes // to propagate the changes to IMAP clients. -func (a *Account) ResetThreading(ctx context.Context, log *mlog.Log, batchSize int, clearIDs bool) (int, error) { +func (a *Account) ResetThreading(ctx context.Context, log mlog.Log, batchSize int, clearIDs bool) (int, error) { // todo: should this send Change events for ThreadMuted and ThreadCollapsed? worth it? var lastID int64 @@ -147,13 +148,13 @@ func (a *Account) ResetThreading(ctx context.Context, log *mlog.Log, batchSize i Envelope *message.Envelope } if err := json.Unmarshal(m.ParsedBuf, &part); err != nil { - log.Errorx("unmarshal json parsedbuf for setting message-id, skipping", err, mlog.Field("msgid", m.ID)) + log.Errorx("unmarshal json parsedbuf for setting message-id, skipping", err, slog.Int64("msgid", m.ID)) } else { m.MessageID = "" if part.Envelope != nil && part.Envelope.MessageID != "" { s, _, err := message.MessageIDCanonical(part.Envelope.MessageID) if err != nil { - log.Debugx("parsing message-id, skipping", err, mlog.Field("msgid", m.ID), mlog.Field("messageid", part.Envelope.MessageID)) + log.Debugx("parsing message-id, skipping", err, slog.Int64("msgid", m.ID), slog.String("messageid", part.Envelope.MessageID)) } m.MessageID = s } @@ -231,7 +232,7 @@ func (a *Account) ResetThreading(ctx context.Context, log *mlog.Log, batchSize i // Does not set Seen flag for muted threads. // // Progress is written to progressWriter, every 100k messages. -func (a *Account) AssignThreads(ctx context.Context, log *mlog.Log, txOpt *bstore.Tx, startMessageID int64, batchSize int, progressWriter io.Writer) error { +func (a *Account) AssignThreads(ctx context.Context, log mlog.Log, txOpt *bstore.Tx, startMessageID int64, batchSize int, progressWriter io.Writer) error { // We use a more basic version of the thread-matching algorithm describe in: // ../rfc/5256:443 // The algorithm assumes you'll select messages, then group into threads. We normally do @@ -294,7 +295,7 @@ func (a *Account) AssignThreads(ctx context.Context, log *mlog.Log, txOpt *bstor refids, err := message.ReferencedIDs(references, inReplyTo) if err != nil { - log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID)) + log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, slog.Int64("msgid", m.ID)) } for i := len(refids) - 1; i >= 0; i-- { @@ -527,7 +528,7 @@ func (a *Account) AssignThreads(ctx context.Context, log *mlog.Log, txOpt *bstor } nassigned += n if nassigned%100000 == 0 { - log.Debug("assigning threads, progress", mlog.Field("count", nassigned), mlog.Field("unresolved", len(pending))) + log.Debug("assigning threads, progress", slog.Int("count", nassigned), slog.Int("unresolved", len(pending))) if _, err := fmt.Fprintf(progressWriter, "assigning threads, progress: %d messages\n", nassigned); err != nil { return fmt.Errorf("writing progress: %v", err) } @@ -537,7 +538,7 @@ func (a *Account) AssignThreads(ctx context.Context, log *mlog.Log, txOpt *bstor return fmt.Errorf("writing progress: %v", err) } - log.Debug("assigning threads, mostly done, finishing with resolving of cyclic messages", mlog.Field("count", nassigned), mlog.Field("unresolved", len(pending))) + log.Debug("assigning threads, mostly done, finishing with resolving of cyclic messages", slog.Int("count", nassigned), slog.Int("unresolved", len(pending))) if _, err := fmt.Fprintf(progressWriter, "assigning threads, resolving %d cyclic pending message-ids\n", len(pending)); err != nil { return fmt.Errorf("writing progress: %v", err) @@ -723,8 +724,8 @@ func lookupThreadMessageSubject(tx *bstore.Tx, m Message, subjectBase string) (* return &tm, nil } -func upgradeThreads(ctx context.Context, acc *Account, up *Upgrade) error { - log := xlog.Fields(mlog.Field("account", acc.Name)) +func upgradeThreads(ctx context.Context, log mlog.Log, acc *Account, up *Upgrade) error { + log = log.With(slog.String("account", acc.Name)) if up.Threads == 0 { // Step 1 in the threads upgrade is storing the canonicalized Message-ID for each @@ -745,7 +746,7 @@ func upgradeThreads(ctx context.Context, acc *Account, up *Upgrade) error { up.Threads = 0 return fmt.Errorf("saving upgrade process while upgrading account to threads storage, step 1/2: %w", err) } - log.Info("upgrading account for threading, step 1/2: completed", mlog.Field("duration", time.Since(t0)), mlog.Field("messages", total)) + log.Info("upgrading account for threading, step 1/2: completed", slog.Duration("duration", time.Since(t0)), slog.Int("messages", total)) } if up.Threads == 1 { @@ -765,7 +766,7 @@ func upgradeThreads(ctx context.Context, acc *Account, up *Upgrade) error { up.Threads = 1 return fmt.Errorf("saving upgrade process for thread storage, step 2/2: %w", err) } - log.Info("upgrading account for threading, step 2/2: completed", mlog.Field("duration", time.Since(t0))) + log.Info("upgrading account for threading, step 2/2: completed", slog.Duration("duration", time.Since(t0))) } // Note: Not bumping uidvalidity or setting modseq. Clients haven't been able to diff --git a/store/threads_test.go b/store/threads_test.go index e76edc7..fa63e5d 100644 --- a/store/threads_test.go +++ b/store/threads_test.go @@ -15,10 +15,11 @@ import ( ) func TestThreadingUpgrade(t *testing.T) { + log := mlog.New("store", nil) os.RemoveAll("../testdata/store/data") mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf") mox.MustLoadConfig(true, false) - acc, err := OpenAccount("mjl") + acc, err := OpenAccount(log, "mjl") tcheck(t, err, "open account") defer func() { err = acc.Close() @@ -26,12 +27,10 @@ func TestThreadingUpgrade(t *testing.T) { }() defer Switchboard()() - log := mlog.New("store") - // New account already has threading. Add some messages, check the threading. deliver := func(recv time.Time, s string, expThreadID int64) Message { t.Helper() - f, err := CreateMessageTemp("account-test") + f, err := CreateMessageTemp(log, "account-test") tcheck(t, err, "temp file") defer os.Remove(f.Name()) defer f.Close() @@ -141,7 +140,7 @@ func TestThreadingUpgrade(t *testing.T) { tcheck(t, err, "closing db") // Open the account again, that should get the account upgraded. Wait for upgrade to finish. - acc, err = OpenAccount("mjl") + acc, err = OpenAccount(log, "mjl") tcheck(t, err, "open account") err = acc.ThreadingWait(log) tcheck(t, err, "wait for threading") diff --git a/store/tmp.go b/store/tmp.go index 7456f21..8c7fbcf 100644 --- a/store/tmp.go +++ b/store/tmp.go @@ -3,6 +3,7 @@ package store import ( "os" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" ) @@ -12,7 +13,7 @@ import ( // responsible for closing and possibly removing the file. The caller should ensure // the contents of the file are synced to disk before attempting to deliver the // message. -func CreateMessageTemp(pattern string) (*os.File, error) { +func CreateMessageTemp(log mlog.Log, pattern string) (*os.File, error) { dir := mox.DataDirPath("tmp") os.MkdirAll(dir, 0770) f, err := os.CreateTemp(dir, pattern) @@ -22,7 +23,7 @@ func CreateMessageTemp(pattern string) (*os.File, error) { err = f.Chmod(0660) if err != nil { xerr := f.Close() - xlog.Check(xerr, "closing temp message file after chmod error") + log.Check(xerr, "closing temp message file after chmod error") return nil, err } return f, err diff --git a/store/train.go b/store/train.go index 803faea..b1e96b3 100644 --- a/store/train.go +++ b/store/train.go @@ -7,6 +7,8 @@ import ( "os" "path/filepath" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/config" @@ -22,7 +24,7 @@ var ErrNoJunkFilter = errors.New("junkfilter: not configured") // If the account does not have a junk filter enabled, ErrNotConfigured is returned. // Do not forget to save the filter after modifying, and to always close the filter when done. // An empty filter is initialized on first access of the filter. -func (a *Account) OpenJunkFilter(ctx context.Context, log *mlog.Log) (*junk.Filter, *config.JunkFilter, error) { +func (a *Account) OpenJunkFilter(ctx context.Context, log mlog.Log) (*junk.Filter, *config.JunkFilter, error) { conf, ok := mox.Conf.Account(a.Name) if !ok { return nil, nil, ErrAccountUnknown @@ -46,7 +48,7 @@ func (a *Account) OpenJunkFilter(ctx context.Context, log *mlog.Log) (*junk.Filt // RetrainMessages (un)trains messages, if relevant given their flags. Updates // m.TrainedJunk after retraining. -func (a *Account) RetrainMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, msgs []Message, absentOK bool) (rerr error) { +func (a *Account) RetrainMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, msgs []Message, absentOK bool) (rerr error) { if len(msgs) == 0 { return nil } @@ -86,7 +88,7 @@ func (a *Account) RetrainMessages(ctx context.Context, log *mlog.Log, tx *bstore // RetrainMessage untrains and/or trains a message, if relevant given m.TrainedJunk // and m.Junk/m.Notjunk. Updates m.TrainedJunk after retraining. -func (a *Account) RetrainMessage(ctx context.Context, log *mlog.Log, tx *bstore.Tx, jf *junk.Filter, m *Message, absentOK bool) error { +func (a *Account) RetrainMessage(ctx context.Context, log mlog.Log, tx *bstore.Tx, jf *junk.Filter, m *Message, absentOK bool) error { untrain := m.TrainedJunk != nil untrainJunk := untrain && *m.TrainedJunk train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk) @@ -96,7 +98,7 @@ func (a *Account) RetrainMessage(ctx context.Context, log *mlog.Log, tx *bstore. return nil } - log.Debug("updating junk filter", mlog.Field("untrain", untrain), mlog.Field("untrainjunk", untrainJunk), mlog.Field("train", train), mlog.Field("trainjunk", trainJunk)) + log.Debug("updating junk filter", slog.Bool("untrain", untrain), slog.Bool("untrainjunk", untrainJunk), slog.Bool("train", train), slog.Bool("trainjunk", trainJunk)) mr := a.MessageReader(*m) defer func() { @@ -112,7 +114,7 @@ func (a *Account) RetrainMessage(ctx context.Context, log *mlog.Log, tx *bstore. words, err := jf.ParseMessage(p) if err != nil { - log.Errorx("parsing message for updating junk filter", err, mlog.Field("parse", "")) + log.Errorx("parsing message for updating junk filter", err, slog.Any("parse", "")) return nil } @@ -138,7 +140,7 @@ func (a *Account) RetrainMessage(ctx context.Context, log *mlog.Log, tx *bstore. // TrainMessage trains the junk filter based on the current m.Junk/m.Notjunk flags, // disregarding m.TrainedJunk and not updating that field. -func (a *Account) TrainMessage(ctx context.Context, log *mlog.Log, jf *junk.Filter, m Message) (bool, error) { +func (a *Account) TrainMessage(ctx context.Context, log mlog.Log, jf *junk.Filter, m Message) (bool, error) { if !m.Junk && !m.Notjunk || (m.Junk && m.Notjunk) { return false, nil } @@ -157,7 +159,7 @@ func (a *Account) TrainMessage(ctx context.Context, log *mlog.Log, jf *junk.Filt words, err := jf.ParseMessage(p) if err != nil { - log.Errorx("parsing message for updating junk filter", err, mlog.Field("parse", "")) + log.Errorx("parsing message for updating junk filter", err, slog.Any("parse", "")) return false, nil } diff --git a/subjectpass/subjectpass.go b/subjectpass/subjectpass.go index 5caa29b..208c79a 100644 --- a/subjectpass/subjectpass.go +++ b/subjectpass/subjectpass.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -20,8 +22,6 @@ import ( "github.com/mjl-/mox/smtp" ) -var log = mlog.New("subjectpass") - var ( metricGenerate = promauto.NewCounter( prometheus.CounterOpts{ @@ -54,9 +54,11 @@ var Explanation = "Your message resembles spam. If your email is legitimate, ple // Generate generates a token that is valid for "mailFrom", starting from "tm" // and signed with "key". // The token is of the form: (pass:) -func Generate(mailFrom smtp.Address, key []byte, tm time.Time) string { +func Generate(elog *slog.Logger, mailFrom smtp.Address, key []byte, tm time.Time) string { + log := mlog.New("subjectpass", elog) + metricGenerate.Inc() - log.Debug("subjectpass generate", mlog.Field("mailfrom", mailFrom)) + log.Debug("subjectpass generate", slog.Any("mailfrom", mailFrom)) // We discard the lower 8 bits of the time, we can do with less precision. t := tm.Unix() @@ -76,7 +78,9 @@ func Generate(mailFrom smtp.Address, key []byte, tm time.Time) string { // Verify parses "message" and checks if it includes a subjectpass token in its // Subject header that is still valid (within "period") and signed with "key". -func Verify(log *mlog.Log, r io.ReaderAt, key []byte, period time.Duration) (rerr error) { +func Verify(elog *slog.Logger, r io.ReaderAt, key []byte, period time.Duration) (rerr error) { + log := mlog.New("subjectpass", elog) + var token string defer func() { @@ -86,10 +90,10 @@ func Verify(log *mlog.Log, r io.ReaderAt, key []byte, period time.Duration) (rer } metricVerify.WithLabelValues(result).Inc() - log.Debugx("subjectpass verify result", rerr, mlog.Field("token", token), mlog.Field("period", period)) + log.Debugx("subjectpass verify result", rerr, slog.String("token", token), slog.Duration("period", period)) }() - p, err := message.Parse(log, true, r) + p, err := message.Parse(log.Logger, true, r) if err != nil { return fmt.Errorf("%w: parse message: %s", ErrMessage, err) } diff --git a/subjectpass/subjectpass_test.go b/subjectpass/subjectpass_test.go index a50a628..7e64e71 100644 --- a/subjectpass/subjectpass_test.go +++ b/subjectpass/subjectpass_test.go @@ -11,25 +11,25 @@ import ( "github.com/mjl-/mox/smtp" ) -var xlog = mlog.New("subjectpass") - func TestSubjectPass(t *testing.T) { + log := mlog.New("subjectpass", nil) + key := []byte("secret token") addr, _ := smtp.ParseAddress("mox@mox.example") - sig := Generate(addr, key, time.Now()) + sig := Generate(log.Logger, addr, key, time.Now()) message := fmt.Sprintf("From: \r\nSubject: let me in %s\r\n\r\nthe message", sig) - if err := Verify(xlog, strings.NewReader(message), key, time.Hour); err != nil { + if err := Verify(log.Logger, strings.NewReader(message), key, time.Hour); err != nil { t.Fatalf("verifyPassToken: %s", err) } - if err := Verify(xlog, strings.NewReader(message), []byte("bad key"), time.Hour); err == nil { + if err := Verify(log.Logger, strings.NewReader(message), []byte("bad key"), time.Hour); err == nil { t.Fatalf("verifyPassToken did not fail") } - sig = Generate(addr, key, time.Now().Add(-time.Hour-257)) + sig = Generate(log.Logger, addr, key, time.Now().Add(-time.Hour-257)) message = fmt.Sprintf("From: \r\nSubject: let me in %s\r\n\r\nthe message", sig) - if err := Verify(xlog, strings.NewReader(message), key, time.Hour); !errors.Is(err, ErrExpired) { + if err := Verify(log.Logger, strings.NewReader(message), key, time.Hour); !errors.Is(err, ErrExpired) { t.Fatalf("verifyPassToken should have expired") } } diff --git a/tlsrpt/lookup.go b/tlsrpt/lookup.go index 7bfecf4..26b633c 100644 --- a/tlsrpt/lookup.go +++ b/tlsrpt/lookup.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -13,8 +15,6 @@ import ( "github.com/mjl-/mox/mlog" ) -var xlog = mlog.New("tlsrpt") - var ( metricLookup = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -35,8 +35,8 @@ var ( // Lookup looks up a TLSRPT DNS TXT record for domain at "_smtp._tls." and // parses it. -func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) { - log := xlog.WithContext(ctx) +func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rrecord *Record, rtxt string, rerr error) { + log := mlog.New("tlsrpt", elog) start := time.Now() defer func() { result := "ok" @@ -54,7 +54,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rrec } } metricLookup.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("tlsrpt lookup result", rerr, mlog.Field("domain", domain), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) + log.Debugx("tlsrpt lookup result", rerr, slog.Any("domain", domain), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start))) }() name := "_smtp._tls." + domain.ASCII + "." diff --git a/tlsrpt/lookup_test.go b/tlsrpt/lookup_test.go index edcce8f..73ca3bf 100644 --- a/tlsrpt/lookup_test.go +++ b/tlsrpt/lookup_test.go @@ -7,9 +7,11 @@ import ( "testing" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" ) func TestLookup(t *testing.T) { + log := mlog.New("tlsrpt", nil) resolver := dns.MockResolver{ TXT: map[string][]string{ "_smtp._tls.basic.example.": {"v=TLSRPTv1; rua=mailto:tlsrpt@basic.example"}, @@ -27,7 +29,7 @@ func TestLookup(t *testing.T) { t.Helper() d := dns.Domain{ASCII: domain} - record, _, err := Lookup(context.Background(), resolver, d) + record, _, err := Lookup(context.Background(), log.Logger, resolver, d) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("lookup, got err %#v, expected %#v", err, expErr) } diff --git a/tlsrpt/report.go b/tlsrpt/report.go index 72ec31a..ca6dbf2 100644 --- a/tlsrpt/report.go +++ b/tlsrpt/report.go @@ -17,6 +17,7 @@ import ( "time" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/mjl-/adns" @@ -368,9 +369,11 @@ func Parse(r io.Reader) (*Report, error) { // ParseMessage parses a Report from a mail message. // The maximum size of the message is 15MB, the maximum size of the // decompressed report is 20MB. -func ParseMessage(log *mlog.Log, r io.ReaderAt) (*Report, error) { +func ParseMessage(elog *slog.Logger, r io.ReaderAt) (*Report, error) { + log := mlog.New("tlsrpt", elog) + // ../rfc/8460:905 - p, err := message.Parse(log, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024}) + p, err := message.Parse(log.Logger, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024}) if err != nil { return nil, fmt.Errorf("parsing mail message: %s", err) } @@ -381,7 +384,7 @@ func ParseMessage(log *mlog.Log, r io.ReaderAt) (*Report, error) { return parseMessageReport(log, p, allow) } -func parseMessageReport(log *mlog.Log, p message.Part, allow bool) (*Report, error) { +func parseMessageReport(log mlog.Log, p message.Part, allow bool) (*Report, error) { if p.MediaType != "MULTIPART" { if !allow { return nil, ErrNoReport @@ -390,7 +393,7 @@ func parseMessageReport(log *mlog.Log, p message.Part, allow bool) (*Report, err } for { - sp, err := p.ParseNextPart(log) + sp, err := p.ParseNextPart(log.Logger) if err == io.EOF { return nil, ErrNoReport } diff --git a/tlsrpt/report_test.go b/tlsrpt/report_test.go index 99206d2..bbaa0f0 100644 --- a/tlsrpt/report_test.go +++ b/tlsrpt/report_test.go @@ -14,8 +14,12 @@ import ( "strings" "testing" "time" + + "github.com/mjl-/mox/mlog" ) +var pkglog = mlog.New("tlsrpt", nil) + const reportJSON = `{ "organization-name": "Company-X", "date-range": { @@ -118,19 +122,19 @@ func TestReport(t *testing.T) { t.Fatalf("parsing report: %s", err) } - if _, err := ParseMessage(xlog, strings.NewReader(tlsrptMessage)); err != nil { + if _, err := ParseMessage(pkglog.Logger, strings.NewReader(tlsrptMessage)); err != nil { t.Fatalf("parsing TLSRPT from message: %s", err) } - if _, err := ParseMessage(xlog, strings.NewReader(tlsrptMessage2)); err != nil { + if _, err := ParseMessage(pkglog.Logger, strings.NewReader(tlsrptMessage2)); err != nil { t.Fatalf("parsing TLSRPT from message: %s", err) } - if _, err := ParseMessage(xlog, strings.NewReader(strings.ReplaceAll(tlsrptMessage, "multipart/report", "multipart/related"))); err != ErrNoReport { + if _, err := ParseMessage(pkglog.Logger, strings.NewReader(strings.ReplaceAll(tlsrptMessage, "multipart/report", "multipart/related"))); err != ErrNoReport { t.Fatalf("got err %v, expected ErrNoReport", err) } - if _, err := ParseMessage(xlog, strings.NewReader(strings.ReplaceAll(tlsrptMessage, "application/tlsrpt+json", "application/json"))); err != ErrNoReport { + if _, err := ParseMessage(pkglog.Logger, strings.NewReader(strings.ReplaceAll(tlsrptMessage, "application/tlsrpt+json", "application/json"))); err != ErrNoReport { t.Fatalf("got err %v, expected ErrNoReport", err) } @@ -143,7 +147,7 @@ func TestReport(t *testing.T) { if err != nil { t.Fatalf("open %q: %s", file, err) } - if _, err := ParseMessage(xlog, f); err != nil { + if _, err := ParseMessage(pkglog.Logger, f); err != nil { t.Fatalf("parsing TLSRPT from message %q: %s", file.Name(), err) } f.Close() @@ -312,6 +316,6 @@ func fakeCert(t *testing.T, name string, expired bool) tls.Certificate { func FuzzParseMessage(f *testing.F) { f.Add(tlsrptMessage) f.Fuzz(func(t *testing.T, s string) { - ParseMessage(xlog, strings.NewReader(s)) + ParseMessage(pkglog.Logger, strings.NewReader(s)) }) } diff --git a/tlsrptdb/db.go b/tlsrptdb/db.go index 2d1f393..1594843 100644 --- a/tlsrptdb/db.go +++ b/tlsrptdb/db.go @@ -10,8 +10,6 @@ import ( ) var ( - xlog = mlog.New("tlsrptdb") - ReportDBTypes = []any{TLSReportRecord{}} ReportDB *bstore.DB mutex sync.Mutex @@ -34,9 +32,10 @@ func Init() error { // Close closes the database connections. func Close() { + log := mlog.New("tlsrptdb", nil) if ResultDB != nil { err := ResultDB.Close() - xlog.Check(err, "closing result database") + log.Check(err, "closing result database") ResultDB = nil } @@ -44,7 +43,7 @@ func Close() { defer mutex.Unlock() if ReportDB != nil { err := ReportDB.Close() - xlog.Check(err, "closing report database") + log.Check(err, "closing report database") ReportDB = nil } } diff --git a/tlsrptdb/report.go b/tlsrptdb/report.go index 0c668ed..56a8c3e 100644 --- a/tlsrptdb/report.go +++ b/tlsrptdb/report.go @@ -8,6 +8,8 @@ import ( "path/filepath" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -81,9 +83,7 @@ func reportDB(ctx context.Context) (rdb *bstore.DB, rerr error) { // domain. Only reports for known domains are added to the database. // // Prometheus metrics are updated only for configured domains. -func AddReport(ctx context.Context, verifiedFromDomain dns.Domain, mailFrom string, hostReport bool, r *tlsrpt.Report) error { - log := xlog.WithContext(ctx) - +func AddReport(ctx context.Context, log mlog.Log, verifiedFromDomain dns.Domain, mailFrom string, hostReport bool, r *tlsrpt.Report) error { db, err := reportDB(ctx) if err != nil { return err @@ -103,14 +103,14 @@ func AddReport(ctx context.Context, verifiedFromDomain dns.Domain, mailFrom stri // coalesce TLS results for different policy domains in a single report. d, err := dns.ParseDomain(pp.Domain) if err != nil { - log.Errorx("invalid domain in tls report", err, mlog.Field("domain", pp.Domain), mlog.Field("mailfrom", mailFrom)) + log.Errorx("invalid domain in tls report", err, slog.Any("domain", pp.Domain), slog.String("mailfrom", mailFrom)) continue } if hostReport && d != mox.Conf.Static.HostnameDomain { - log.Info("unknown mail host policy domain in tls report, not storing", mlog.Field("domain", d), mlog.Field("mailfrom", mailFrom)) + log.Info("unknown mail host policy domain in tls report, not storing", slog.Any("domain", d), slog.String("mailfrom", mailFrom)) return fmt.Errorf("unknown mail host policy domain") } else if _, ok := mox.Conf.Domain(d); !hostReport && !ok { - log.Info("unknown recipient policy domain in tls report, not storing", mlog.Field("domain", d), mlog.Field("mailfrom", mailFrom)) + log.Info("unknown recipient policy domain in tls report, not storing", slog.Any("domain", d), slog.String("mailfrom", mailFrom)) return fmt.Errorf("unknown recipient policy domain") } if reportdom != zerodom && d != reportdom { diff --git a/tlsrptdb/report_test.go b/tlsrptdb/report_test.go index 64553be..5052003 100644 --- a/tlsrptdb/report_test.go +++ b/tlsrptdb/report_test.go @@ -11,11 +11,13 @@ import ( "github.com/mjl-/mox/config" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/tlsrpt" ) var ctxbg = context.Background() +var pkglog = mlog.New("tlsrptdb", nil) const reportJSON = `{ "organization-name": "Company-X", @@ -89,12 +91,12 @@ func TestReport(t *testing.T) { if err != nil { t.Fatalf("open %q: %s", file, err) } - report, err := tlsrpt.ParseMessage(xlog, f) + report, err := tlsrpt.ParseMessage(pkglog.Logger, f) f.Close() if err != nil { t.Fatalf("parsing TLSRPT from message %q: %s", file.Name(), err) } - if err := AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, report); err != nil { + if err := AddReport(ctxbg, pkglog, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, report); err != nil { t.Fatalf("adding report to database: %s", err) } } @@ -102,7 +104,7 @@ func TestReport(t *testing.T) { report, err := tlsrpt.Parse(strings.NewReader(reportJSON)) if err != nil { t.Fatalf("parsing report: %v", err) - } else if err := AddReport(ctxbg, dns.Domain{ASCII: "company-y.example"}, "tlsrpt@company-y.example", false, report); err != nil { + } else if err := AddReport(ctxbg, pkglog, dns.Domain{ASCII: "company-y.example"}, "tlsrpt@company-y.example", false, report); err != nil { t.Fatalf("adding report to database: %s", err) } diff --git a/tlsrptsend/send.go b/tlsrptsend/send.go index b2f6a51..e80094b 100644 --- a/tlsrptsend/send.go +++ b/tlsrptsend/send.go @@ -30,6 +30,7 @@ import ( "time" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -80,13 +81,13 @@ var jitteredTimeUntil = func(t time.Time) time.Duration { // reports. Reports are sent spread out over a 4 hour period. func Start(resolver dns.Resolver) { go func() { - log := mlog.New("tlsrptsend") + log := mlog.New("tlsrptsend", nil) defer func() { // In case of panic don't take the whole program down. x := recover() if x != nil { - log.Error("recover from panic", mlog.Field("panic", x)) + log.Error("recover from panic", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Tlsrptdb) } @@ -117,7 +118,7 @@ func Start(resolver dns.Resolver) { log.Check(err, "removing stale tls results from database") clog := log.WithCid(mox.Cid()) - clog.Info("sending tls reports", mlog.Field("day", dayUTC)) + clog.Info("sending tls reports", slog.String("day", dayUTC)) if err := sendReports(ctx, clog, resolver, db, dayUTC, endUTC); err != nil { clog.Errorx("sending tls reports", err) metricReportError.Inc() @@ -159,7 +160,7 @@ var sleepBetween = func(ctx context.Context, between time.Duration) (ok bool) { // sendReports gathers all policy domains that have results that should receive a // TLS report and sends a report to each if their TLSRPT DNS record has reporting // addresses. -func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, dayUTC string, endTimeUTC time.Time) error { +func sendReports(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, dayUTC string, endTimeUTC time.Time) error { type key struct { policyDomain string dayUTC string @@ -228,14 +229,14 @@ func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db * // In case of panic don't take the whole program down. x := recover() if x != nil { - log.Error("unhandled panic in tlsrptsend sendReports", mlog.Field("panic", x)) + log.Error("unhandled panic in tlsrptsend sendReports", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Tlsrptdb) } }() defer wg.Done() - rlog := log.WithCid(mox.Cid()).Fields(mlog.Field("policydomain", k.policyDomain), mlog.Field("daytutc", k.dayUTC), mlog.Field("isrcptdom", isRcptDom)) + rlog := log.WithCid(mox.Cid()).With(slog.String("policydomain", k.policyDomain), slog.String("daytutc", k.dayUTC), slog.Bool("isrcptdom", isRcptDom)) rlog.Info("looking to send tls report for domain") cleanup, err := sendReportDomain(ctx, rlog, resolver, db, endTimeUTC, isRcptDom, k.policyDomain, k.dayUTC) if err != nil { @@ -279,7 +280,7 @@ func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db * // replaceable for testing. var queueAdd = queue.Add -func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endUTC time.Time, isRcptDom bool, policyDomain, dayUTC string) (cleanup bool, rerr error) { +func sendReportDomain(ctx context.Context, log mlog.Log, resolver dns.Resolver, db *bstore.DB, endUTC time.Time, isRcptDom bool, policyDomain, dayUTC string) (cleanup bool, rerr error) { polDom, err := dns.ParseDomain(policyDomain) if err != nil { return false, fmt.Errorf("parsing policy domain for sending tls reports: %v", err) @@ -324,7 +325,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, }() // Get TLSRPT record. If there are no reporting addresses, we're not going to send at all. - record, _, err := tlsrpt.Lookup(ctx, resolver, polDom) + record, _, err := tlsrpt.Lookup(ctx, log.Logger, resolver, polDom) if err != nil { // If there is no TLSRPT record, that's fine, we'll remove what we tracked. if errors.Is(err, tlsrpt.ErrNoRecord) { @@ -341,14 +342,14 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, for _, s := range l { u, err := url.Parse(string(s)) if err != nil { - log.Debugx("parsing rua uri in tlsrpt dns record, ignoring", err, mlog.Field("rua", s)) + log.Debugx("parsing rua uri in tlsrpt dns record, ignoring", err, slog.Any("rua", s)) continue } if u.Scheme == "mailto" { addr, err := smtp.ParseAddress(u.Opaque) if err != nil { - log.Debugx("parsing mailto uri in tlsrpt record rua value, ignoring", err, mlog.Field("rua", s)) + log.Debugx("parsing mailto uri in tlsrpt record rua value, ignoring", err, slog.Any("rua", s)) continue } recipients = append(recipients, message.NameAddress{Address: addr}) @@ -365,9 +366,9 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, // them. // ../rfc/8460:320 ../rfc/8460:1055 // todo spec: would be good to have clearer distinction between "report" (JSON) and "report message" (message with report attachment, that can be DKIM signed). propose sending report message over https that includes DKIM signature so authenticity can be verified and the report used. ../rfc/8460:310 - log.Debug("https scheme in rua uri in tlsrpt record, ignoring since they will likey not be used to due lack of authentication", mlog.Field("rua", s)) + log.Debug("https scheme in rua uri in tlsrpt record, ignoring since they will likey not be used to due lack of authentication", slog.Any("rua", s)) } else { - log.Debug("unknown scheme in rua uri in tlsrpt record, ignoring", mlog.Field("rua", s)) + log.Debug("unknown scheme in rua uri in tlsrpt record, ignoring", slog.Any("rua", s)) } } } @@ -472,7 +473,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, log.Info("sending tls report") - reportFile, err := store.CreateMessageTemp("tlsreportout") + reportFile, err := store.CreateMessageTemp(log, "tlsreportout") if err != nil { return false, fmt.Errorf("creating temporary file for outgoing tls report: %v", err) } @@ -492,7 +493,7 @@ func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, return false, fmt.Errorf("writing tls report as json with gzip: %v", err) } - msgf, err := store.CreateMessageTemp("tlsreportmsgout") + msgf, err := store.CreateMessageTemp(log, "tlsreportmsgout") if err != nil { return false, fmt.Errorf("creating temporary message file with outgoing tls report: %v", err) } @@ -579,7 +580,7 @@ Period: %s - %s UTC return false, fmt.Errorf("querying suppress list: %v", err) } if exists { - log.Info("suppressing outgoing tls report", mlog.Field("reportingaddress", rcpt.Address)) + log.Info("suppressing outgoing tls report", slog.Any("reportingaddress", rcpt.Address)) continue } @@ -601,7 +602,7 @@ Period: %s - %s UTC } else { queued = true tempError = false - log.Debug("tls report queued", mlog.Field("recipient", rcpt)) + log.Debug("tls report queued", slog.Any("recipient", rcpt)) metricReport.Inc() } } @@ -613,7 +614,7 @@ Period: %s - %s UTC return true, nil } -func composeMessage(ctx context.Context, log *mlog.Log, mf *os.File, policyDomain dns.Domain, confDKIM config.DKIM, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { +func composeMessage(ctx context.Context, log mlog.Log, mf *os.File, policyDomain dns.Domain, confDKIM config.DKIM, fromAddr smtp.Address, recipients []message.NameAddress, subject, text, filename string, reportFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { xc := message.NewComposer(mf, 100*1024*1024) defer func() { x := recover() @@ -693,7 +694,7 @@ func composeMessage(ctx context.Context, log *mlog.Log, mf *os.File, policyDomai } confDKIM.Selectors = selectors - dkimHeader, err := dkim.Sign(ctx, fromAddr.Localpart, fromAddr.Domain, confDKIM, smtputf8, mf) + dkimHeader, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fromAddr.Domain, confDKIM, smtputf8, mf) xc.Checkf(err, "dkim-signing report message") return dkimHeader, xc.Has8bit, xc.SMTPUTF8, messageID, nil diff --git a/tlsrptsend/send_test.go b/tlsrptsend/send_test.go index 33536c8..a288c4f 100644 --- a/tlsrptsend/send_test.go +++ b/tlsrptsend/send_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/bstore" "github.com/mjl-/mox/dns" @@ -39,7 +41,7 @@ func tcompare(t *testing.T, got, expect any) { } func TestSendReports(t *testing.T) { - mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) + mlog.SetConfig(map[string]slog.Level{"": mlog.LevelDebug}) os.RemoveAll("../testdata/tlsrptsend/data") mox.Context = ctxbg @@ -411,7 +413,7 @@ func TestSendReports(t *testing.T) { var mutex sync.Mutex var index int - queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error { + queueAdd = func(ctx context.Context, log mlog.Log, qm *queue.Msg, msgFile *os.File) error { mutex.Lock() defer mutex.Unlock() @@ -423,7 +425,7 @@ func TestSendReports(t *testing.T) { err = os.WriteFile(p, append(append([]byte{}, qm.MsgPrefix...), buf...), 0600) tcheckf(t, err, "write report message") - report, err := tlsrpt.ParseMessage(log, msgFile) + report, err := tlsrpt.ParseMessage(log.Logger, msgFile) tcheckf(t, err, "parsing generated report message") addr := qm.Recipient().String() diff --git a/updates/updates.go b/updates/updates.go index 6287b57..95a875b 100644 --- a/updates/updates.go +++ b/updates/updates.go @@ -21,6 +21,8 @@ import ( "strings" "time" + "golang.org/x/exp/slog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -30,8 +32,6 @@ import ( "github.com/mjl-/mox/moxio" ) -var xlog = mlog.New("updates") - var ( metricLookup = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -81,8 +81,8 @@ type Changelog struct { // Lookup looks up the updates DNS TXT record at "_updates." and returns // the parsed form. -func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rversion Version, rrecord *Record, rerr error) { - log := xlog.WithContext(ctx) +func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain) (rversion Version, rrecord *Record, rerr error) { + log := mlog.New("updates", elog) start := time.Now() defer func() { var result = "ok" @@ -90,7 +90,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rver result = "error" } metricLookup.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("updates lookup result", rerr, mlog.Field("domain", domain), mlog.Field("version", rversion), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) + log.Debugx("updates lookup result", rerr, slog.Any("domain", domain), slog.Any("version", rversion), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start))) }() nctx, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -132,8 +132,8 @@ func Lookup(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (rver // error is returned. // // A changelog can be maximum 1 MB. -func FetchChangelog(ctx context.Context, baseURL string, base Version, pubKey []byte) (changelog *Changelog, rerr error) { - log := xlog.WithContext(ctx) +func FetchChangelog(ctx context.Context, elog *slog.Logger, baseURL string, base Version, pubKey []byte) (changelog *Changelog, rerr error) { + log := mlog.New("updates", elog) start := time.Now() defer func() { var result = "ok" @@ -141,7 +141,7 @@ func FetchChangelog(ctx context.Context, baseURL string, base Version, pubKey [] result = "error" } metricFetchChangelog.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) - log.Debugx("updates fetch changelog result", rerr, mlog.Field("baseurl", baseURL), mlog.Field("base", base), mlog.Field("duration", time.Since(start))) + log.Debugx("updates fetch changelog result", rerr, slog.String("baseurl", baseURL), slog.Any("base", base), slog.Duration("duration", time.Since(start))) }() url := baseURL + "?from=" + base.String() @@ -156,7 +156,7 @@ func FetchChangelog(ctx context.Context, baseURL string, base Version, pubKey [] if resp == nil { resp = &http.Response{StatusCode: 0} } - metrics.HTTPClientObserve(ctx, "updates", req.Method, resp.StatusCode, err, start) + metrics.HTTPClientObserve(ctx, log, "updates", req.Method, resp.StatusCode, err, start) if err != nil { return nil, fmt.Errorf("%w: making http request: %s", ErrChangelogFetch, err) } @@ -182,20 +182,20 @@ func FetchChangelog(ctx context.Context, baseURL string, base Version, pubKey [] // Check checks for an updated version through DNS and fetches a // changelog if so. -func Check(ctx context.Context, resolver dns.Resolver, domain dns.Domain, lastKnown Version, changelogBaseURL string, pubKey []byte) (rversion Version, rrecord *Record, changelog *Changelog, rerr error) { - log := xlog.WithContext(ctx) +func Check(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain dns.Domain, lastKnown Version, changelogBaseURL string, pubKey []byte) (rversion Version, rrecord *Record, changelog *Changelog, rerr error) { + log := mlog.New("updates", elog) start := time.Now() defer func() { - log.Debugx("updates check result", rerr, mlog.Field("domain", domain), mlog.Field("lastknown", lastKnown), mlog.Field("changelogbaseurl", changelogBaseURL), mlog.Field("version", rversion), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start))) + log.Debugx("updates check result", rerr, slog.Any("domain", domain), slog.Any("lastknown", lastKnown), slog.String("changelogbaseurl", changelogBaseURL), slog.Any("version", rversion), slog.Any("record", rrecord), slog.Duration("duration", time.Since(start))) }() - latest, record, err := Lookup(ctx, resolver, domain) + latest, record, err := Lookup(ctx, log.Logger, resolver, domain) if err != nil { return latest, record, nil, err } if latest.After(lastKnown) { - changelog, err = FetchChangelog(ctx, changelogBaseURL, lastKnown, pubKey) + changelog, err = FetchChangelog(ctx, log.Logger, changelogBaseURL, lastKnown, pubKey) } return latest, record, changelog, err } diff --git a/updates/updates_test.go b/updates/updates_test.go index 4e4b644..32120ed 100644 --- a/updates/updates_test.go +++ b/updates/updates_test.go @@ -6,16 +6,19 @@ import ( "encoding/json" "errors" "io" - "log" + golog "log" "net/http" "net/http/httptest" "reflect" "testing" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" ) func TestUpdates(t *testing.T) { + log := mlog.New("updates", nil) + resolver := dns.MockResolver{ TXT: map[string][]string{ "_updates.mox.example.": {"v=UPDATES0; l=v0.0.1"}, @@ -39,7 +42,7 @@ func TestUpdates(t *testing.T) { d, _ := dns.ParseDomain(dom) expv, _ := ParseVersion(expVersion) - version, record, err := Lookup(context.Background(), resolver, d) + version, record, err := Lookup(context.Background(), log.Logger, resolver, d) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("lookup: got err %v, expected %v", err, expErr) } @@ -87,14 +90,14 @@ func TestUpdates(t *testing.T) { } }) s := httptest.NewUnstartedServer(mux) - s.Config.ErrorLog = log.New(io.Discard, "", 0) + s.Config.ErrorLog = golog.New(io.Discard, "", 0) s.Start() defer s.Close() if baseURL == "" { baseURL = s.URL } - changelog, err := FetchChangelog(context.Background(), baseURL, version, pubKey) + changelog, err := FetchChangelog(context.Background(), log.Logger, baseURL, version, pubKey) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("fetch changelog: got err %v, expected %v", err, expErr) } @@ -129,14 +132,14 @@ func TestUpdates(t *testing.T) { } }) s := httptest.NewUnstartedServer(mux) - s.Config.ErrorLog = log.New(io.Discard, "", 0) + s.Config.ErrorLog = golog.New(io.Discard, "", 0) s.Start() defer s.Close() if baseURL == "" { baseURL = s.URL } - version, record, changelog, err := Check(context.Background(), resolver, dns.Domain{ASCII: dom}, base, baseURL, pubKey) + version, record, changelog, err := Check(context.Background(), log.Logger, resolver, dns.Domain{ASCII: dom}, base, baseURL, pubKey) if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) { t.Fatalf("check: got err %v, expected %v", err, expErr) } diff --git a/vendor/golang.org/x/exp/slog/attr.go b/vendor/golang.org/x/exp/slog/attr.go new file mode 100644 index 0000000..a180d0e --- /dev/null +++ b/vendor/golang.org/x/exp/slog/attr.go @@ -0,0 +1,102 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "fmt" + "time" +) + +// An Attr is a key-value pair. +type Attr struct { + Key string + Value Value +} + +// String returns an Attr for a string value. +func String(key, value string) Attr { + return Attr{key, StringValue(value)} +} + +// Int64 returns an Attr for an int64. +func Int64(key string, value int64) Attr { + return Attr{key, Int64Value(value)} +} + +// Int converts an int to an int64 and returns +// an Attr with that value. +func Int(key string, value int) Attr { + return Int64(key, int64(value)) +} + +// Uint64 returns an Attr for a uint64. +func Uint64(key string, v uint64) Attr { + return Attr{key, Uint64Value(v)} +} + +// Float64 returns an Attr for a floating-point number. +func Float64(key string, v float64) Attr { + return Attr{key, Float64Value(v)} +} + +// Bool returns an Attr for a bool. +func Bool(key string, v bool) Attr { + return Attr{key, BoolValue(v)} +} + +// Time returns an Attr for a time.Time. +// It discards the monotonic portion. +func Time(key string, v time.Time) Attr { + return Attr{key, TimeValue(v)} +} + +// Duration returns an Attr for a time.Duration. +func Duration(key string, v time.Duration) Attr { + return Attr{key, DurationValue(v)} +} + +// Group returns an Attr for a Group Value. +// The first argument is the key; the remaining arguments +// are converted to Attrs as in [Logger.Log]. +// +// Use Group to collect several key-value pairs under a single +// key on a log line, or as the result of LogValue +// in order to log a single value as multiple Attrs. +func Group(key string, args ...any) Attr { + return Attr{key, GroupValue(argsToAttrSlice(args)...)} +} + +func argsToAttrSlice(args []any) []Attr { + var ( + attr Attr + attrs []Attr + ) + for len(args) > 0 { + attr, args = argsToAttr(args) + attrs = append(attrs, attr) + } + return attrs +} + +// Any returns an Attr for the supplied value. +// See [Value.AnyValue] for how values are treated. +func Any(key string, value any) Attr { + return Attr{key, AnyValue(value)} +} + +// Equal reports whether a and b have equal keys and values. +func (a Attr) Equal(b Attr) bool { + return a.Key == b.Key && a.Value.Equal(b.Value) +} + +func (a Attr) String() string { + return fmt.Sprintf("%s=%s", a.Key, a.Value) +} + +// isEmpty reports whether a has an empty key and a nil value. +// That can be written as Attr{} or Any("", nil). +func (a Attr) isEmpty() bool { + return a.Key == "" && a.Value.num == 0 && a.Value.any == nil +} diff --git a/vendor/golang.org/x/exp/slog/doc.go b/vendor/golang.org/x/exp/slog/doc.go new file mode 100644 index 0000000..4beaf86 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/doc.go @@ -0,0 +1,316 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package slog provides structured logging, +in which log records include a message, +a severity level, and various other attributes +expressed as key-value pairs. + +It defines a type, [Logger], +which provides several methods (such as [Logger.Info] and [Logger.Error]) +for reporting events of interest. + +Each Logger is associated with a [Handler]. +A Logger output method creates a [Record] from the method arguments +and passes it to the Handler, which decides how to handle it. +There is a default Logger accessible through top-level functions +(such as [Info] and [Error]) that call the corresponding Logger methods. + +A log record consists of a time, a level, a message, and a set of key-value +pairs, where the keys are strings and the values may be of any type. +As an example, + + slog.Info("hello", "count", 3) + +creates a record containing the time of the call, +a level of Info, the message "hello", and a single +pair with key "count" and value 3. + +The [Info] top-level function calls the [Logger.Info] method on the default Logger. +In addition to [Logger.Info], there are methods for Debug, Warn and Error levels. +Besides these convenience methods for common levels, +there is also a [Logger.Log] method which takes the level as an argument. +Each of these methods has a corresponding top-level function that uses the +default logger. + +The default handler formats the log record's message, time, level, and attributes +as a string and passes it to the [log] package. + + 2022/11/08 15:28:26 INFO hello count=3 + +For more control over the output format, create a logger with a different handler. +This statement uses [New] to create a new logger with a TextHandler +that writes structured records in text form to standard error: + + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) + +[TextHandler] output is a sequence of key=value pairs, easily and unambiguously +parsed by machine. This statement: + + logger.Info("hello", "count", 3) + +produces this output: + + time=2022-11-08T15:28:26.000-05:00 level=INFO msg=hello count=3 + +The package also provides [JSONHandler], whose output is line-delimited JSON: + + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + logger.Info("hello", "count", 3) + +produces this output: + + {"time":"2022-11-08T15:28:26.000000000-05:00","level":"INFO","msg":"hello","count":3} + +Both [TextHandler] and [JSONHandler] can be configured with [HandlerOptions]. +There are options for setting the minimum level (see Levels, below), +displaying the source file and line of the log call, and +modifying attributes before they are logged. + +Setting a logger as the default with + + slog.SetDefault(logger) + +will cause the top-level functions like [Info] to use it. +[SetDefault] also updates the default logger used by the [log] package, +so that existing applications that use [log.Printf] and related functions +will send log records to the logger's handler without needing to be rewritten. + +Some attributes are common to many log calls. +For example, you may wish to include the URL or trace identifier of a server request +with all log events arising from the request. +Rather than repeat the attribute with every log call, you can use [Logger.With] +to construct a new Logger containing the attributes: + + logger2 := logger.With("url", r.URL) + +The arguments to With are the same key-value pairs used in [Logger.Info]. +The result is a new Logger with the same handler as the original, but additional +attributes that will appear in the output of every call. + +# Levels + +A [Level] is an integer representing the importance or severity of a log event. +The higher the level, the more severe the event. +This package defines constants for the most common levels, +but any int can be used as a level. + +In an application, you may wish to log messages only at a certain level or greater. +One common configuration is to log messages at Info or higher levels, +suppressing debug logging until it is needed. +The built-in handlers can be configured with the minimum level to output by +setting [HandlerOptions.Level]. +The program's `main` function typically does this. +The default value is LevelInfo. + +Setting the [HandlerOptions.Level] field to a [Level] value +fixes the handler's minimum level throughout its lifetime. +Setting it to a [LevelVar] allows the level to be varied dynamically. +A LevelVar holds a Level and is safe to read or write from multiple +goroutines. +To vary the level dynamically for an entire program, first initialize +a global LevelVar: + + var programLevel = new(slog.LevelVar) // Info by default + +Then use the LevelVar to construct a handler, and make it the default: + + h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: programLevel}) + slog.SetDefault(slog.New(h)) + +Now the program can change its logging level with a single statement: + + programLevel.Set(slog.LevelDebug) + +# Groups + +Attributes can be collected into groups. +A group has a name that is used to qualify the names of its attributes. +How this qualification is displayed depends on the handler. +[TextHandler] separates the group and attribute names with a dot. +[JSONHandler] treats each group as a separate JSON object, with the group name as the key. + +Use [Group] to create a Group attribute from a name and a list of key-value pairs: + + slog.Group("request", + "method", r.Method, + "url", r.URL) + +TextHandler would display this group as + + request.method=GET request.url=http://example.com + +JSONHandler would display it as + + "request":{"method":"GET","url":"http://example.com"} + +Use [Logger.WithGroup] to qualify all of a Logger's output +with a group name. Calling WithGroup on a Logger results in a +new Logger with the same Handler as the original, but with all +its attributes qualified by the group name. + +This can help prevent duplicate attribute keys in large systems, +where subsystems might use the same keys. +Pass each subsystem a different Logger with its own group name so that +potential duplicates are qualified: + + logger := slog.Default().With("id", systemID) + parserLogger := logger.WithGroup("parser") + parseInput(input, parserLogger) + +When parseInput logs with parserLogger, its keys will be qualified with "parser", +so even if it uses the common key "id", the log line will have distinct keys. + +# Contexts + +Some handlers may wish to include information from the [context.Context] that is +available at the call site. One example of such information +is the identifier for the current span when tracing is enabled. + +The [Logger.Log] and [Logger.LogAttrs] methods take a context as a first +argument, as do their corresponding top-level functions. + +Although the convenience methods on Logger (Info and so on) and the +corresponding top-level functions do not take a context, the alternatives ending +in "Context" do. For example, + + slog.InfoContext(ctx, "message") + +It is recommended to pass a context to an output method if one is available. + +# Attrs and Values + +An [Attr] is a key-value pair. The Logger output methods accept Attrs as well as +alternating keys and values. The statement + + slog.Info("hello", slog.Int("count", 3)) + +behaves the same as + + slog.Info("hello", "count", 3) + +There are convenience constructors for [Attr] such as [Int], [String], and [Bool] +for common types, as well as the function [Any] for constructing Attrs of any +type. + +The value part of an Attr is a type called [Value]. +Like an [any], a Value can hold any Go value, +but it can represent typical values, including all numbers and strings, +without an allocation. + +For the most efficient log output, use [Logger.LogAttrs]. +It is similar to [Logger.Log] but accepts only Attrs, not alternating +keys and values; this allows it, too, to avoid allocation. + +The call + + logger.LogAttrs(nil, slog.LevelInfo, "hello", slog.Int("count", 3)) + +is the most efficient way to achieve the same output as + + slog.Info("hello", "count", 3) + +# Customizing a type's logging behavior + +If a type implements the [LogValuer] interface, the [Value] returned from its LogValue +method is used for logging. You can use this to control how values of the type +appear in logs. For example, you can redact secret information like passwords, +or gather a struct's fields in a Group. See the examples under [LogValuer] for +details. + +A LogValue method may return a Value that itself implements [LogValuer]. The [Value.Resolve] +method handles these cases carefully, avoiding infinite loops and unbounded recursion. +Handler authors and others may wish to use Value.Resolve instead of calling LogValue directly. + +# Wrapping output methods + +The logger functions use reflection over the call stack to find the file name +and line number of the logging call within the application. This can produce +incorrect source information for functions that wrap slog. For instance, if you +define this function in file mylog.go: + + func Infof(format string, args ...any) { + slog.Default().Info(fmt.Sprintf(format, args...)) + } + +and you call it like this in main.go: + + Infof(slog.Default(), "hello, %s", "world") + +then slog will report the source file as mylog.go, not main.go. + +A correct implementation of Infof will obtain the source location +(pc) and pass it to NewRecord. +The Infof function in the package-level example called "wrapping" +demonstrates how to do this. + +# Working with Records + +Sometimes a Handler will need to modify a Record +before passing it on to another Handler or backend. +A Record contains a mixture of simple public fields (e.g. Time, Level, Message) +and hidden fields that refer to state (such as attributes) indirectly. This +means that modifying a simple copy of a Record (e.g. by calling +[Record.Add] or [Record.AddAttrs] to add attributes) +may have unexpected effects on the original. +Before modifying a Record, use [Clone] to +create a copy that shares no state with the original, +or create a new Record with [NewRecord] +and build up its Attrs by traversing the old ones with [Record.Attrs]. + +# Performance considerations + +If profiling your application demonstrates that logging is taking significant time, +the following suggestions may help. + +If many log lines have a common attribute, use [Logger.With] to create a Logger with +that attribute. The built-in handlers will format that attribute only once, at the +call to [Logger.With]. The [Handler] interface is designed to allow that optimization, +and a well-written Handler should take advantage of it. + +The arguments to a log call are always evaluated, even if the log event is discarded. +If possible, defer computation so that it happens only if the value is actually logged. +For example, consider the call + + slog.Info("starting request", "url", r.URL.String()) // may compute String unnecessarily + +The URL.String method will be called even if the logger discards Info-level events. +Instead, pass the URL directly: + + slog.Info("starting request", "url", &r.URL) // calls URL.String only if needed + +The built-in [TextHandler] will call its String method, but only +if the log event is enabled. +Avoiding the call to String also preserves the structure of the underlying value. +For example [JSONHandler] emits the components of the parsed URL as a JSON object. +If you want to avoid eagerly paying the cost of the String call +without causing the handler to potentially inspect the structure of the value, +wrap the value in a fmt.Stringer implementation that hides its Marshal methods. + +You can also use the [LogValuer] interface to avoid unnecessary work in disabled log +calls. Say you need to log some expensive value: + + slog.Debug("frobbing", "value", computeExpensiveValue(arg)) + +Even if this line is disabled, computeExpensiveValue will be called. +To avoid that, define a type implementing LogValuer: + + type expensive struct { arg int } + + func (e expensive) LogValue() slog.Value { + return slog.AnyValue(computeExpensiveValue(e.arg)) + } + +Then use a value of that type in log calls: + + slog.Debug("frobbing", "value", expensive{arg}) + +Now computeExpensiveValue will only be called when the line is enabled. + +The built-in handlers acquire a lock before calling [io.Writer.Write] +to ensure that each record is written in one piece. User-defined +handlers are responsible for their own locking. +*/ +package slog diff --git a/vendor/golang.org/x/exp/slog/handler.go b/vendor/golang.org/x/exp/slog/handler.go new file mode 100644 index 0000000..74f8873 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/handler.go @@ -0,0 +1,559 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "context" + "fmt" + "io" + "strconv" + "sync" + "time" + + "golang.org/x/exp/slices" + "golang.org/x/exp/slog/internal/buffer" +) + +// A Handler handles log records produced by a Logger.. +// +// A typical handler may print log records to standard error, +// or write them to a file or database, or perhaps augment them +// with additional attributes and pass them on to another handler. +// +// Any of the Handler's methods may be called concurrently with itself +// or with other methods. It is the responsibility of the Handler to +// manage this concurrency. +// +// Users of the slog package should not invoke Handler methods directly. +// They should use the methods of [Logger] instead. +type Handler interface { + // Enabled reports whether the handler handles records at the given level. + // The handler ignores records whose level is lower. + // It is called early, before any arguments are processed, + // to save effort if the log event should be discarded. + // If called from a Logger method, the first argument is the context + // passed to that method, or context.Background() if nil was passed + // or the method does not take a context. + // The context is passed so Enabled can use its values + // to make a decision. + Enabled(context.Context, Level) bool + + // Handle handles the Record. + // It will only be called when Enabled returns true. + // The Context argument is as for Enabled. + // It is present solely to provide Handlers access to the context's values. + // Canceling the context should not affect record processing. + // (Among other things, log messages may be necessary to debug a + // cancellation-related problem.) + // + // Handle methods that produce output should observe the following rules: + // - If r.Time is the zero time, ignore the time. + // - If r.PC is zero, ignore it. + // - Attr's values should be resolved. + // - If an Attr's key and value are both the zero value, ignore the Attr. + // This can be tested with attr.Equal(Attr{}). + // - If a group's key is empty, inline the group's Attrs. + // - If a group has no Attrs (even if it has a non-empty key), + // ignore it. + Handle(context.Context, Record) error + + // WithAttrs returns a new Handler whose attributes consist of + // both the receiver's attributes and the arguments. + // The Handler owns the slice: it may retain, modify or discard it. + WithAttrs(attrs []Attr) Handler + + // WithGroup returns a new Handler with the given group appended to + // the receiver's existing groups. + // The keys of all subsequent attributes, whether added by With or in a + // Record, should be qualified by the sequence of group names. + // + // How this qualification happens is up to the Handler, so long as + // this Handler's attribute keys differ from those of another Handler + // with a different sequence of group names. + // + // A Handler should treat WithGroup as starting a Group of Attrs that ends + // at the end of the log event. That is, + // + // logger.WithGroup("s").LogAttrs(level, msg, slog.Int("a", 1), slog.Int("b", 2)) + // + // should behave like + // + // logger.LogAttrs(level, msg, slog.Group("s", slog.Int("a", 1), slog.Int("b", 2))) + // + // If the name is empty, WithGroup returns the receiver. + WithGroup(name string) Handler +} + +type defaultHandler struct { + ch *commonHandler + // log.Output, except for testing + output func(calldepth int, message string) error +} + +func newDefaultHandler(output func(int, string) error) *defaultHandler { + return &defaultHandler{ + ch: &commonHandler{json: false}, + output: output, + } +} + +func (*defaultHandler) Enabled(_ context.Context, l Level) bool { + return l >= LevelInfo +} + +// Collect the level, attributes and message in a string and +// write it with the default log.Logger. +// Let the log.Logger handle time and file/line. +func (h *defaultHandler) Handle(ctx context.Context, r Record) error { + buf := buffer.New() + buf.WriteString(r.Level.String()) + buf.WriteByte(' ') + buf.WriteString(r.Message) + state := h.ch.newHandleState(buf, true, " ", nil) + defer state.free() + state.appendNonBuiltIns(r) + + // skip [h.output, defaultHandler.Handle, handlerWriter.Write, log.Output] + return h.output(4, buf.String()) +} + +func (h *defaultHandler) WithAttrs(as []Attr) Handler { + return &defaultHandler{h.ch.withAttrs(as), h.output} +} + +func (h *defaultHandler) WithGroup(name string) Handler { + return &defaultHandler{h.ch.withGroup(name), h.output} +} + +// HandlerOptions are options for a TextHandler or JSONHandler. +// A zero HandlerOptions consists entirely of default values. +type HandlerOptions struct { + // AddSource causes the handler to compute the source code position + // of the log statement and add a SourceKey attribute to the output. + AddSource bool + + // Level reports the minimum record level that will be logged. + // The handler discards records with lower levels. + // If Level is nil, the handler assumes LevelInfo. + // The handler calls Level.Level for each record processed; + // to adjust the minimum level dynamically, use a LevelVar. + Level Leveler + + // ReplaceAttr is called to rewrite each non-group attribute before it is logged. + // The attribute's value has been resolved (see [Value.Resolve]). + // If ReplaceAttr returns an Attr with Key == "", the attribute is discarded. + // + // The built-in attributes with keys "time", "level", "source", and "msg" + // are passed to this function, except that time is omitted + // if zero, and source is omitted if AddSource is false. + // + // The first argument is a list of currently open groups that contain the + // Attr. It must not be retained or modified. ReplaceAttr is never called + // for Group attributes, only their contents. For example, the attribute + // list + // + // Int("a", 1), Group("g", Int("b", 2)), Int("c", 3) + // + // results in consecutive calls to ReplaceAttr with the following arguments: + // + // nil, Int("a", 1) + // []string{"g"}, Int("b", 2) + // nil, Int("c", 3) + // + // ReplaceAttr can be used to change the default keys of the built-in + // attributes, convert types (for example, to replace a `time.Time` with the + // integer seconds since the Unix epoch), sanitize personal information, or + // remove attributes from the output. + ReplaceAttr func(groups []string, a Attr) Attr +} + +// Keys for "built-in" attributes. +const ( + // TimeKey is the key used by the built-in handlers for the time + // when the log method is called. The associated Value is a [time.Time]. + TimeKey = "time" + // LevelKey is the key used by the built-in handlers for the level + // of the log call. The associated value is a [Level]. + LevelKey = "level" + // MessageKey is the key used by the built-in handlers for the + // message of the log call. The associated value is a string. + MessageKey = "msg" + // SourceKey is the key used by the built-in handlers for the source file + // and line of the log call. The associated value is a string. + SourceKey = "source" +) + +type commonHandler struct { + json bool // true => output JSON; false => output text + opts HandlerOptions + preformattedAttrs []byte + groupPrefix string // for text: prefix of groups opened in preformatting + groups []string // all groups started from WithGroup + nOpenGroups int // the number of groups opened in preformattedAttrs + mu sync.Mutex + w io.Writer +} + +func (h *commonHandler) clone() *commonHandler { + // We can't use assignment because we can't copy the mutex. + return &commonHandler{ + json: h.json, + opts: h.opts, + preformattedAttrs: slices.Clip(h.preformattedAttrs), + groupPrefix: h.groupPrefix, + groups: slices.Clip(h.groups), + nOpenGroups: h.nOpenGroups, + w: h.w, + } +} + +// enabled reports whether l is greater than or equal to the +// minimum level. +func (h *commonHandler) enabled(l Level) bool { + minLevel := LevelInfo + if h.opts.Level != nil { + minLevel = h.opts.Level.Level() + } + return l >= minLevel +} + +func (h *commonHandler) withAttrs(as []Attr) *commonHandler { + h2 := h.clone() + // Pre-format the attributes as an optimization. + prefix := buffer.New() + defer prefix.Free() + prefix.WriteString(h.groupPrefix) + state := h2.newHandleState((*buffer.Buffer)(&h2.preformattedAttrs), false, "", prefix) + defer state.free() + if len(h2.preformattedAttrs) > 0 { + state.sep = h.attrSep() + } + state.openGroups() + for _, a := range as { + state.appendAttr(a) + } + // Remember the new prefix for later keys. + h2.groupPrefix = state.prefix.String() + // Remember how many opened groups are in preformattedAttrs, + // so we don't open them again when we handle a Record. + h2.nOpenGroups = len(h2.groups) + return h2 +} + +func (h *commonHandler) withGroup(name string) *commonHandler { + if name == "" { + return h + } + h2 := h.clone() + h2.groups = append(h2.groups, name) + return h2 +} + +func (h *commonHandler) handle(r Record) error { + state := h.newHandleState(buffer.New(), true, "", nil) + defer state.free() + if h.json { + state.buf.WriteByte('{') + } + // Built-in attributes. They are not in a group. + stateGroups := state.groups + state.groups = nil // So ReplaceAttrs sees no groups instead of the pre groups. + rep := h.opts.ReplaceAttr + // time + if !r.Time.IsZero() { + key := TimeKey + val := r.Time.Round(0) // strip monotonic to match Attr behavior + if rep == nil { + state.appendKey(key) + state.appendTime(val) + } else { + state.appendAttr(Time(key, val)) + } + } + // level + key := LevelKey + val := r.Level + if rep == nil { + state.appendKey(key) + state.appendString(val.String()) + } else { + state.appendAttr(Any(key, val)) + } + // source + if h.opts.AddSource { + state.appendAttr(Any(SourceKey, r.source())) + } + key = MessageKey + msg := r.Message + if rep == nil { + state.appendKey(key) + state.appendString(msg) + } else { + state.appendAttr(String(key, msg)) + } + state.groups = stateGroups // Restore groups passed to ReplaceAttrs. + state.appendNonBuiltIns(r) + state.buf.WriteByte('\n') + + h.mu.Lock() + defer h.mu.Unlock() + _, err := h.w.Write(*state.buf) + return err +} + +func (s *handleState) appendNonBuiltIns(r Record) { + // preformatted Attrs + if len(s.h.preformattedAttrs) > 0 { + s.buf.WriteString(s.sep) + s.buf.Write(s.h.preformattedAttrs) + s.sep = s.h.attrSep() + } + // Attrs in Record -- unlike the built-in ones, they are in groups started + // from WithGroup. + s.prefix = buffer.New() + defer s.prefix.Free() + s.prefix.WriteString(s.h.groupPrefix) + s.openGroups() + r.Attrs(func(a Attr) bool { + s.appendAttr(a) + return true + }) + if s.h.json { + // Close all open groups. + for range s.h.groups { + s.buf.WriteByte('}') + } + // Close the top-level object. + s.buf.WriteByte('}') + } +} + +// attrSep returns the separator between attributes. +func (h *commonHandler) attrSep() string { + if h.json { + return "," + } + return " " +} + +// handleState holds state for a single call to commonHandler.handle. +// The initial value of sep determines whether to emit a separator +// before the next key, after which it stays true. +type handleState struct { + h *commonHandler + buf *buffer.Buffer + freeBuf bool // should buf be freed? + sep string // separator to write before next key + prefix *buffer.Buffer // for text: key prefix + groups *[]string // pool-allocated slice of active groups, for ReplaceAttr +} + +var groupPool = sync.Pool{New: func() any { + s := make([]string, 0, 10) + return &s +}} + +func (h *commonHandler) newHandleState(buf *buffer.Buffer, freeBuf bool, sep string, prefix *buffer.Buffer) handleState { + s := handleState{ + h: h, + buf: buf, + freeBuf: freeBuf, + sep: sep, + prefix: prefix, + } + if h.opts.ReplaceAttr != nil { + s.groups = groupPool.Get().(*[]string) + *s.groups = append(*s.groups, h.groups[:h.nOpenGroups]...) + } + return s +} + +func (s *handleState) free() { + if s.freeBuf { + s.buf.Free() + } + if gs := s.groups; gs != nil { + *gs = (*gs)[:0] + groupPool.Put(gs) + } +} + +func (s *handleState) openGroups() { + for _, n := range s.h.groups[s.h.nOpenGroups:] { + s.openGroup(n) + } +} + +// Separator for group names and keys. +const keyComponentSep = '.' + +// openGroup starts a new group of attributes +// with the given name. +func (s *handleState) openGroup(name string) { + if s.h.json { + s.appendKey(name) + s.buf.WriteByte('{') + s.sep = "" + } else { + s.prefix.WriteString(name) + s.prefix.WriteByte(keyComponentSep) + } + // Collect group names for ReplaceAttr. + if s.groups != nil { + *s.groups = append(*s.groups, name) + } +} + +// closeGroup ends the group with the given name. +func (s *handleState) closeGroup(name string) { + if s.h.json { + s.buf.WriteByte('}') + } else { + (*s.prefix) = (*s.prefix)[:len(*s.prefix)-len(name)-1 /* for keyComponentSep */] + } + s.sep = s.h.attrSep() + if s.groups != nil { + *s.groups = (*s.groups)[:len(*s.groups)-1] + } +} + +// appendAttr appends the Attr's key and value using app. +// It handles replacement and checking for an empty key. +// after replacement). +func (s *handleState) appendAttr(a Attr) { + if rep := s.h.opts.ReplaceAttr; rep != nil && a.Value.Kind() != KindGroup { + var gs []string + if s.groups != nil { + gs = *s.groups + } + // Resolve before calling ReplaceAttr, so the user doesn't have to. + a.Value = a.Value.Resolve() + a = rep(gs, a) + } + a.Value = a.Value.Resolve() + // Elide empty Attrs. + if a.isEmpty() { + return + } + // Special case: Source. + if v := a.Value; v.Kind() == KindAny { + if src, ok := v.Any().(*Source); ok { + if s.h.json { + a.Value = src.group() + } else { + a.Value = StringValue(fmt.Sprintf("%s:%d", src.File, src.Line)) + } + } + } + if a.Value.Kind() == KindGroup { + attrs := a.Value.Group() + // Output only non-empty groups. + if len(attrs) > 0 { + // Inline a group with an empty key. + if a.Key != "" { + s.openGroup(a.Key) + } + for _, aa := range attrs { + s.appendAttr(aa) + } + if a.Key != "" { + s.closeGroup(a.Key) + } + } + } else { + s.appendKey(a.Key) + s.appendValue(a.Value) + } +} + +func (s *handleState) appendError(err error) { + s.appendString(fmt.Sprintf("!ERROR:%v", err)) +} + +func (s *handleState) appendKey(key string) { + s.buf.WriteString(s.sep) + if s.prefix != nil { + // TODO: optimize by avoiding allocation. + s.appendString(string(*s.prefix) + key) + } else { + s.appendString(key) + } + if s.h.json { + s.buf.WriteByte(':') + } else { + s.buf.WriteByte('=') + } + s.sep = s.h.attrSep() +} + +func (s *handleState) appendString(str string) { + if s.h.json { + s.buf.WriteByte('"') + *s.buf = appendEscapedJSONString(*s.buf, str) + s.buf.WriteByte('"') + } else { + // text + if needsQuoting(str) { + *s.buf = strconv.AppendQuote(*s.buf, str) + } else { + s.buf.WriteString(str) + } + } +} + +func (s *handleState) appendValue(v Value) { + var err error + if s.h.json { + err = appendJSONValue(s, v) + } else { + err = appendTextValue(s, v) + } + if err != nil { + s.appendError(err) + } +} + +func (s *handleState) appendTime(t time.Time) { + if s.h.json { + appendJSONTime(s, t) + } else { + writeTimeRFC3339Millis(s.buf, t) + } +} + +// This takes half the time of Time.AppendFormat. +func writeTimeRFC3339Millis(buf *buffer.Buffer, t time.Time) { + year, month, day := t.Date() + buf.WritePosIntWidth(year, 4) + buf.WriteByte('-') + buf.WritePosIntWidth(int(month), 2) + buf.WriteByte('-') + buf.WritePosIntWidth(day, 2) + buf.WriteByte('T') + hour, min, sec := t.Clock() + buf.WritePosIntWidth(hour, 2) + buf.WriteByte(':') + buf.WritePosIntWidth(min, 2) + buf.WriteByte(':') + buf.WritePosIntWidth(sec, 2) + ns := t.Nanosecond() + buf.WriteByte('.') + buf.WritePosIntWidth(ns/1e6, 3) + _, offsetSeconds := t.Zone() + if offsetSeconds == 0 { + buf.WriteByte('Z') + } else { + offsetMinutes := offsetSeconds / 60 + if offsetMinutes < 0 { + buf.WriteByte('-') + offsetMinutes = -offsetMinutes + } else { + buf.WriteByte('+') + } + buf.WritePosIntWidth(offsetMinutes/60, 2) + buf.WriteByte(':') + buf.WritePosIntWidth(offsetMinutes%60, 2) + } +} diff --git a/vendor/golang.org/x/exp/slog/internal/buffer/buffer.go b/vendor/golang.org/x/exp/slog/internal/buffer/buffer.go new file mode 100644 index 0000000..7786c16 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/internal/buffer/buffer.go @@ -0,0 +1,84 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package buffer provides a pool-allocated byte buffer. +package buffer + +import ( + "sync" +) + +// Buffer adapted from go/src/fmt/print.go +type Buffer []byte + +// Having an initial size gives a dramatic speedup. +var bufPool = sync.Pool{ + New: func() any { + b := make([]byte, 0, 1024) + return (*Buffer)(&b) + }, +} + +func New() *Buffer { + return bufPool.Get().(*Buffer) +} + +func (b *Buffer) Free() { + // To reduce peak allocation, return only smaller buffers to the pool. + const maxBufferSize = 16 << 10 + if cap(*b) <= maxBufferSize { + *b = (*b)[:0] + bufPool.Put(b) + } +} + +func (b *Buffer) Reset() { + *b = (*b)[:0] +} + +func (b *Buffer) Write(p []byte) (int, error) { + *b = append(*b, p...) + return len(p), nil +} + +func (b *Buffer) WriteString(s string) { + *b = append(*b, s...) +} + +func (b *Buffer) WriteByte(c byte) { + *b = append(*b, c) +} + +func (b *Buffer) WritePosInt(i int) { + b.WritePosIntWidth(i, 0) +} + +// WritePosIntWidth writes non-negative integer i to the buffer, padded on the left +// by zeroes to the given width. Use a width of 0 to omit padding. +func (b *Buffer) WritePosIntWidth(i, width int) { + // Cheap integer to fixed-width decimal ASCII. + // Copied from log/log.go. + + if i < 0 { + panic("negative int") + } + + // Assemble decimal in reverse order. + var bb [20]byte + bp := len(bb) - 1 + for i >= 10 || width > 1 { + width-- + q := i / 10 + bb[bp] = byte('0' + i - q*10) + bp-- + i = q + } + // i < 10 + bb[bp] = byte('0' + i) + b.Write(bb[bp:]) +} + +func (b *Buffer) String() string { + return string(*b) +} diff --git a/vendor/golang.org/x/exp/slog/internal/ignorepc.go b/vendor/golang.org/x/exp/slog/internal/ignorepc.go new file mode 100644 index 0000000..d125642 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/internal/ignorepc.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +// If IgnorePC is true, do not invoke runtime.Callers to get the pc. +// This is solely for benchmarking the slowdown from runtime.Callers. +var IgnorePC = false diff --git a/vendor/golang.org/x/exp/slog/json_handler.go b/vendor/golang.org/x/exp/slog/json_handler.go new file mode 100644 index 0000000..157ada8 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/json_handler.go @@ -0,0 +1,336 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "time" + "unicode/utf8" + + "golang.org/x/exp/slog/internal/buffer" +) + +// JSONHandler is a Handler that writes Records to an io.Writer as +// line-delimited JSON objects. +type JSONHandler struct { + *commonHandler +} + +// NewJSONHandler creates a JSONHandler that writes to w, +// using the given options. +// If opts is nil, the default options are used. +func NewJSONHandler(w io.Writer, opts *HandlerOptions) *JSONHandler { + if opts == nil { + opts = &HandlerOptions{} + } + return &JSONHandler{ + &commonHandler{ + json: true, + w: w, + opts: *opts, + }, + } +} + +// Enabled reports whether the handler handles records at the given level. +// The handler ignores records whose level is lower. +func (h *JSONHandler) Enabled(_ context.Context, level Level) bool { + return h.commonHandler.enabled(level) +} + +// WithAttrs returns a new JSONHandler whose attributes consists +// of h's attributes followed by attrs. +func (h *JSONHandler) WithAttrs(attrs []Attr) Handler { + return &JSONHandler{commonHandler: h.commonHandler.withAttrs(attrs)} +} + +func (h *JSONHandler) WithGroup(name string) Handler { + return &JSONHandler{commonHandler: h.commonHandler.withGroup(name)} +} + +// Handle formats its argument Record as a JSON object on a single line. +// +// If the Record's time is zero, the time is omitted. +// Otherwise, the key is "time" +// and the value is output as with json.Marshal. +// +// If the Record's level is zero, the level is omitted. +// Otherwise, the key is "level" +// and the value of [Level.String] is output. +// +// If the AddSource option is set and source information is available, +// the key is "source" +// and the value is output as "FILE:LINE". +// +// The message's key is "msg". +// +// To modify these or other attributes, or remove them from the output, use +// [HandlerOptions.ReplaceAttr]. +// +// Values are formatted as with an [encoding/json.Encoder] with SetEscapeHTML(false), +// with two exceptions. +// +// First, an Attr whose Value is of type error is formatted as a string, by +// calling its Error method. Only errors in Attrs receive this special treatment, +// not errors embedded in structs, slices, maps or other data structures that +// are processed by the encoding/json package. +// +// Second, an encoding failure does not cause Handle to return an error. +// Instead, the error message is formatted as a string. +// +// Each call to Handle results in a single serialized call to io.Writer.Write. +func (h *JSONHandler) Handle(_ context.Context, r Record) error { + return h.commonHandler.handle(r) +} + +// Adapted from time.Time.MarshalJSON to avoid allocation. +func appendJSONTime(s *handleState, t time.Time) { + if y := t.Year(); y < 0 || y >= 10000 { + // RFC 3339 is clear that years are 4 digits exactly. + // See golang.org/issue/4556#c15 for more discussion. + s.appendError(errors.New("time.Time year outside of range [0,9999]")) + } + s.buf.WriteByte('"') + *s.buf = t.AppendFormat(*s.buf, time.RFC3339Nano) + s.buf.WriteByte('"') +} + +func appendJSONValue(s *handleState, v Value) error { + switch v.Kind() { + case KindString: + s.appendString(v.str()) + case KindInt64: + *s.buf = strconv.AppendInt(*s.buf, v.Int64(), 10) + case KindUint64: + *s.buf = strconv.AppendUint(*s.buf, v.Uint64(), 10) + case KindFloat64: + // json.Marshal is funny about floats; it doesn't + // always match strconv.AppendFloat. So just call it. + // That's expensive, but floats are rare. + if err := appendJSONMarshal(s.buf, v.Float64()); err != nil { + return err + } + case KindBool: + *s.buf = strconv.AppendBool(*s.buf, v.Bool()) + case KindDuration: + // Do what json.Marshal does. + *s.buf = strconv.AppendInt(*s.buf, int64(v.Duration()), 10) + case KindTime: + s.appendTime(v.Time()) + case KindAny: + a := v.Any() + _, jm := a.(json.Marshaler) + if err, ok := a.(error); ok && !jm { + s.appendString(err.Error()) + } else { + return appendJSONMarshal(s.buf, a) + } + default: + panic(fmt.Sprintf("bad kind: %s", v.Kind())) + } + return nil +} + +func appendJSONMarshal(buf *buffer.Buffer, v any) error { + // Use a json.Encoder to avoid escaping HTML. + var bb bytes.Buffer + enc := json.NewEncoder(&bb) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return err + } + bs := bb.Bytes() + buf.Write(bs[:len(bs)-1]) // remove final newline + return nil +} + +// appendEscapedJSONString escapes s for JSON and appends it to buf. +// It does not surround the string in quotation marks. +// +// Modified from encoding/json/encode.go:encodeState.string, +// with escapeHTML set to false. +func appendEscapedJSONString(buf []byte, s string) []byte { + char := func(b byte) { buf = append(buf, b) } + str := func(s string) { buf = append(buf, s...) } + + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if safeSet[b] { + i++ + continue + } + if start < i { + str(s[start:i]) + } + char('\\') + switch b { + case '\\', '"': + char(b) + case '\n': + char('n') + case '\r': + char('r') + case '\t': + char('t') + default: + // This encodes bytes < 0x20 except for \t, \n and \r. + str(`u00`) + char(hex[b>>4]) + char(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + str(s[start:i]) + } + str(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + str(s[start:i]) + } + str(`\u202`) + char(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + str(s[start:]) + } + return buf +} + +var hex = "0123456789abcdef" + +// Copied from encoding/json/tables.go. +// +// safeSet holds the value true if the ASCII character with the given array +// position can be represented inside a JSON string without any further +// escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), and the backslash character ("\"). +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} diff --git a/vendor/golang.org/x/exp/slog/level.go b/vendor/golang.org/x/exp/slog/level.go new file mode 100644 index 0000000..b2365f0 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/level.go @@ -0,0 +1,201 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "errors" + "fmt" + "strconv" + "strings" + "sync/atomic" +) + +// A Level is the importance or severity of a log event. +// The higher the level, the more important or severe the event. +type Level int + +// Level numbers are inherently arbitrary, +// but we picked them to satisfy three constraints. +// Any system can map them to another numbering scheme if it wishes. +// +// First, we wanted the default level to be Info, Since Levels are ints, Info is +// the default value for int, zero. +// + +// Second, we wanted to make it easy to use levels to specify logger verbosity. +// Since a larger level means a more severe event, a logger that accepts events +// with smaller (or more negative) level means a more verbose logger. Logger +// verbosity is thus the negation of event severity, and the default verbosity +// of 0 accepts all events at least as severe as INFO. +// +// Third, we wanted some room between levels to accommodate schemes with named +// levels between ours. For example, Google Cloud Logging defines a Notice level +// between Info and Warn. Since there are only a few of these intermediate +// levels, the gap between the numbers need not be large. Our gap of 4 matches +// OpenTelemetry's mapping. Subtracting 9 from an OpenTelemetry level in the +// DEBUG, INFO, WARN and ERROR ranges converts it to the corresponding slog +// Level range. OpenTelemetry also has the names TRACE and FATAL, which slog +// does not. But those OpenTelemetry levels can still be represented as slog +// Levels by using the appropriate integers. +// +// Names for common levels. +const ( + LevelDebug Level = -4 + LevelInfo Level = 0 + LevelWarn Level = 4 + LevelError Level = 8 +) + +// String returns a name for the level. +// If the level has a name, then that name +// in uppercase is returned. +// If the level is between named values, then +// an integer is appended to the uppercased name. +// Examples: +// +// LevelWarn.String() => "WARN" +// (LevelInfo+2).String() => "INFO+2" +func (l Level) String() string { + str := func(base string, val Level) string { + if val == 0 { + return base + } + return fmt.Sprintf("%s%+d", base, val) + } + + switch { + case l < LevelInfo: + return str("DEBUG", l-LevelDebug) + case l < LevelWarn: + return str("INFO", l-LevelInfo) + case l < LevelError: + return str("WARN", l-LevelWarn) + default: + return str("ERROR", l-LevelError) + } +} + +// MarshalJSON implements [encoding/json.Marshaler] +// by quoting the output of [Level.String]. +func (l Level) MarshalJSON() ([]byte, error) { + // AppendQuote is sufficient for JSON-encoding all Level strings. + // They don't contain any runes that would produce invalid JSON + // when escaped. + return strconv.AppendQuote(nil, l.String()), nil +} + +// UnmarshalJSON implements [encoding/json.Unmarshaler] +// It accepts any string produced by [Level.MarshalJSON], +// ignoring case. +// It also accepts numeric offsets that would result in a different string on +// output. For example, "Error-8" would marshal as "INFO". +func (l *Level) UnmarshalJSON(data []byte) error { + s, err := strconv.Unquote(string(data)) + if err != nil { + return err + } + return l.parse(s) +} + +// MarshalText implements [encoding.TextMarshaler] +// by calling [Level.String]. +func (l Level) MarshalText() ([]byte, error) { + return []byte(l.String()), nil +} + +// UnmarshalText implements [encoding.TextUnmarshaler]. +// It accepts any string produced by [Level.MarshalText], +// ignoring case. +// It also accepts numeric offsets that would result in a different string on +// output. For example, "Error-8" would marshal as "INFO". +func (l *Level) UnmarshalText(data []byte) error { + return l.parse(string(data)) +} + +func (l *Level) parse(s string) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("slog: level string %q: %w", s, err) + } + }() + + name := s + offset := 0 + if i := strings.IndexAny(s, "+-"); i >= 0 { + name = s[:i] + offset, err = strconv.Atoi(s[i:]) + if err != nil { + return err + } + } + switch strings.ToUpper(name) { + case "DEBUG": + *l = LevelDebug + case "INFO": + *l = LevelInfo + case "WARN": + *l = LevelWarn + case "ERROR": + *l = LevelError + default: + return errors.New("unknown name") + } + *l += Level(offset) + return nil +} + +// Level returns the receiver. +// It implements Leveler. +func (l Level) Level() Level { return l } + +// A LevelVar is a Level variable, to allow a Handler level to change +// dynamically. +// It implements Leveler as well as a Set method, +// and it is safe for use by multiple goroutines. +// The zero LevelVar corresponds to LevelInfo. +type LevelVar struct { + val atomic.Int64 +} + +// Level returns v's level. +func (v *LevelVar) Level() Level { + return Level(int(v.val.Load())) +} + +// Set sets v's level to l. +func (v *LevelVar) Set(l Level) { + v.val.Store(int64(l)) +} + +func (v *LevelVar) String() string { + return fmt.Sprintf("LevelVar(%s)", v.Level()) +} + +// MarshalText implements [encoding.TextMarshaler] +// by calling [Level.MarshalText]. +func (v *LevelVar) MarshalText() ([]byte, error) { + return v.Level().MarshalText() +} + +// UnmarshalText implements [encoding.TextUnmarshaler] +// by calling [Level.UnmarshalText]. +func (v *LevelVar) UnmarshalText(data []byte) error { + var l Level + if err := l.UnmarshalText(data); err != nil { + return err + } + v.Set(l) + return nil +} + +// A Leveler provides a Level value. +// +// As Level itself implements Leveler, clients typically supply +// a Level value wherever a Leveler is needed, such as in HandlerOptions. +// Clients who need to vary the level dynamically can provide a more complex +// Leveler implementation such as *LevelVar. +type Leveler interface { + Level() Level +} diff --git a/vendor/golang.org/x/exp/slog/logger.go b/vendor/golang.org/x/exp/slog/logger.go new file mode 100644 index 0000000..e87ec99 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/logger.go @@ -0,0 +1,343 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "context" + "log" + "runtime" + "sync/atomic" + "time" + + "golang.org/x/exp/slog/internal" +) + +var defaultLogger atomic.Value + +func init() { + defaultLogger.Store(New(newDefaultHandler(log.Output))) +} + +// Default returns the default Logger. +func Default() *Logger { return defaultLogger.Load().(*Logger) } + +// SetDefault makes l the default Logger. +// After this call, output from the log package's default Logger +// (as with [log.Print], etc.) will be logged at LevelInfo using l's Handler. +func SetDefault(l *Logger) { + defaultLogger.Store(l) + // If the default's handler is a defaultHandler, then don't use a handleWriter, + // or we'll deadlock as they both try to acquire the log default mutex. + // The defaultHandler will use whatever the log default writer is currently + // set to, which is correct. + // This can occur with SetDefault(Default()). + // See TestSetDefault. + if _, ok := l.Handler().(*defaultHandler); !ok { + capturePC := log.Flags()&(log.Lshortfile|log.Llongfile) != 0 + log.SetOutput(&handlerWriter{l.Handler(), LevelInfo, capturePC}) + log.SetFlags(0) // we want just the log message, no time or location + } +} + +// handlerWriter is an io.Writer that calls a Handler. +// It is used to link the default log.Logger to the default slog.Logger. +type handlerWriter struct { + h Handler + level Level + capturePC bool +} + +func (w *handlerWriter) Write(buf []byte) (int, error) { + if !w.h.Enabled(context.Background(), w.level) { + return 0, nil + } + var pc uintptr + if !internal.IgnorePC && w.capturePC { + // skip [runtime.Callers, w.Write, Logger.Output, log.Print] + var pcs [1]uintptr + runtime.Callers(4, pcs[:]) + pc = pcs[0] + } + + // Remove final newline. + origLen := len(buf) // Report that the entire buf was written. + if len(buf) > 0 && buf[len(buf)-1] == '\n' { + buf = buf[:len(buf)-1] + } + r := NewRecord(time.Now(), w.level, string(buf), pc) + return origLen, w.h.Handle(context.Background(), r) +} + +// A Logger records structured information about each call to its +// Log, Debug, Info, Warn, and Error methods. +// For each call, it creates a Record and passes it to a Handler. +// +// To create a new Logger, call [New] or a Logger method +// that begins "With". +type Logger struct { + handler Handler // for structured logging +} + +func (l *Logger) clone() *Logger { + c := *l + return &c +} + +// Handler returns l's Handler. +func (l *Logger) Handler() Handler { return l.handler } + +// With returns a new Logger that includes the given arguments, converted to +// Attrs as in [Logger.Log]. +// The Attrs will be added to each output from the Logger. +// The new Logger shares the old Logger's context. +// The new Logger's handler is the result of calling WithAttrs on the receiver's +// handler. +func (l *Logger) With(args ...any) *Logger { + c := l.clone() + c.handler = l.handler.WithAttrs(argsToAttrSlice(args)) + return c +} + +// WithGroup returns a new Logger that starts a group. The keys of all +// attributes added to the Logger will be qualified by the given name. +// (How that qualification happens depends on the [Handler.WithGroup] +// method of the Logger's Handler.) +// The new Logger shares the old Logger's context. +// +// The new Logger's handler is the result of calling WithGroup on the receiver's +// handler. +func (l *Logger) WithGroup(name string) *Logger { + c := l.clone() + c.handler = l.handler.WithGroup(name) + return c + +} + +// New creates a new Logger with the given non-nil Handler and a nil context. +func New(h Handler) *Logger { + if h == nil { + panic("nil Handler") + } + return &Logger{handler: h} +} + +// With calls Logger.With on the default logger. +func With(args ...any) *Logger { + return Default().With(args...) +} + +// Enabled reports whether l emits log records at the given context and level. +func (l *Logger) Enabled(ctx context.Context, level Level) bool { + if ctx == nil { + ctx = context.Background() + } + return l.Handler().Enabled(ctx, level) +} + +// NewLogLogger returns a new log.Logger such that each call to its Output method +// dispatches a Record to the specified handler. The logger acts as a bridge from +// the older log API to newer structured logging handlers. +func NewLogLogger(h Handler, level Level) *log.Logger { + return log.New(&handlerWriter{h, level, true}, "", 0) +} + +// Log emits a log record with the current time and the given level and message. +// The Record's Attrs consist of the Logger's attributes followed by +// the Attrs specified by args. +// +// The attribute arguments are processed as follows: +// - If an argument is an Attr, it is used as is. +// - If an argument is a string and this is not the last argument, +// the following argument is treated as the value and the two are combined +// into an Attr. +// - Otherwise, the argument is treated as a value with key "!BADKEY". +func (l *Logger) Log(ctx context.Context, level Level, msg string, args ...any) { + l.log(ctx, level, msg, args...) +} + +// LogAttrs is a more efficient version of [Logger.Log] that accepts only Attrs. +func (l *Logger) LogAttrs(ctx context.Context, level Level, msg string, attrs ...Attr) { + l.logAttrs(ctx, level, msg, attrs...) +} + +// Debug logs at LevelDebug. +func (l *Logger) Debug(msg string, args ...any) { + l.log(nil, LevelDebug, msg, args...) +} + +// DebugContext logs at LevelDebug with the given context. +func (l *Logger) DebugContext(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelDebug, msg, args...) +} + +// DebugCtx logs at LevelDebug with the given context. +// Deprecated: Use Logger.DebugContext. +func (l *Logger) DebugCtx(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelDebug, msg, args...) +} + +// Info logs at LevelInfo. +func (l *Logger) Info(msg string, args ...any) { + l.log(nil, LevelInfo, msg, args...) +} + +// InfoContext logs at LevelInfo with the given context. +func (l *Logger) InfoContext(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelInfo, msg, args...) +} + +// InfoCtx logs at LevelInfo with the given context. +// Deprecated: Use Logger.InfoContext. +func (l *Logger) InfoCtx(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelInfo, msg, args...) +} + +// Warn logs at LevelWarn. +func (l *Logger) Warn(msg string, args ...any) { + l.log(nil, LevelWarn, msg, args...) +} + +// WarnContext logs at LevelWarn with the given context. +func (l *Logger) WarnContext(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelWarn, msg, args...) +} + +// WarnCtx logs at LevelWarn with the given context. +// Deprecated: Use Logger.WarnContext. +func (l *Logger) WarnCtx(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelWarn, msg, args...) +} + +// Error logs at LevelError. +func (l *Logger) Error(msg string, args ...any) { + l.log(nil, LevelError, msg, args...) +} + +// ErrorContext logs at LevelError with the given context. +func (l *Logger) ErrorContext(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelError, msg, args...) +} + +// ErrorCtx logs at LevelError with the given context. +// Deprecated: Use Logger.ErrorContext. +func (l *Logger) ErrorCtx(ctx context.Context, msg string, args ...any) { + l.log(ctx, LevelError, msg, args...) +} + +// log is the low-level logging method for methods that take ...any. +// It must always be called directly by an exported logging method +// or function, because it uses a fixed call depth to obtain the pc. +func (l *Logger) log(ctx context.Context, level Level, msg string, args ...any) { + if !l.Enabled(ctx, level) { + return + } + var pc uintptr + if !internal.IgnorePC { + var pcs [1]uintptr + // skip [runtime.Callers, this function, this function's caller] + runtime.Callers(3, pcs[:]) + pc = pcs[0] + } + r := NewRecord(time.Now(), level, msg, pc) + r.Add(args...) + if ctx == nil { + ctx = context.Background() + } + _ = l.Handler().Handle(ctx, r) +} + +// logAttrs is like [Logger.log], but for methods that take ...Attr. +func (l *Logger) logAttrs(ctx context.Context, level Level, msg string, attrs ...Attr) { + if !l.Enabled(ctx, level) { + return + } + var pc uintptr + if !internal.IgnorePC { + var pcs [1]uintptr + // skip [runtime.Callers, this function, this function's caller] + runtime.Callers(3, pcs[:]) + pc = pcs[0] + } + r := NewRecord(time.Now(), level, msg, pc) + r.AddAttrs(attrs...) + if ctx == nil { + ctx = context.Background() + } + _ = l.Handler().Handle(ctx, r) +} + +// Debug calls Logger.Debug on the default logger. +func Debug(msg string, args ...any) { + Default().log(nil, LevelDebug, msg, args...) +} + +// DebugContext calls Logger.DebugContext on the default logger. +func DebugContext(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelDebug, msg, args...) +} + +// Info calls Logger.Info on the default logger. +func Info(msg string, args ...any) { + Default().log(nil, LevelInfo, msg, args...) +} + +// InfoContext calls Logger.InfoContext on the default logger. +func InfoContext(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelInfo, msg, args...) +} + +// Warn calls Logger.Warn on the default logger. +func Warn(msg string, args ...any) { + Default().log(nil, LevelWarn, msg, args...) +} + +// WarnContext calls Logger.WarnContext on the default logger. +func WarnContext(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelWarn, msg, args...) +} + +// Error calls Logger.Error on the default logger. +func Error(msg string, args ...any) { + Default().log(nil, LevelError, msg, args...) +} + +// ErrorContext calls Logger.ErrorContext on the default logger. +func ErrorContext(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelError, msg, args...) +} + +// DebugCtx calls Logger.DebugContext on the default logger. +// Deprecated: call DebugContext. +func DebugCtx(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelDebug, msg, args...) +} + +// InfoCtx calls Logger.InfoContext on the default logger. +// Deprecated: call InfoContext. +func InfoCtx(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelInfo, msg, args...) +} + +// WarnCtx calls Logger.WarnContext on the default logger. +// Deprecated: call WarnContext. +func WarnCtx(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelWarn, msg, args...) +} + +// ErrorCtx calls Logger.ErrorContext on the default logger. +// Deprecated: call ErrorContext. +func ErrorCtx(ctx context.Context, msg string, args ...any) { + Default().log(ctx, LevelError, msg, args...) +} + +// Log calls Logger.Log on the default logger. +func Log(ctx context.Context, level Level, msg string, args ...any) { + Default().log(ctx, level, msg, args...) +} + +// LogAttrs calls Logger.LogAttrs on the default logger. +func LogAttrs(ctx context.Context, level Level, msg string, attrs ...Attr) { + Default().logAttrs(ctx, level, msg, attrs...) +} diff --git a/vendor/golang.org/x/exp/slog/noplog.bench b/vendor/golang.org/x/exp/slog/noplog.bench new file mode 100644 index 0000000..ed9296f --- /dev/null +++ b/vendor/golang.org/x/exp/slog/noplog.bench @@ -0,0 +1,36 @@ +goos: linux +goarch: amd64 +pkg: golang.org/x/exp/slog +cpu: Intel(R) Xeon(R) CPU @ 2.20GHz +BenchmarkNopLog/attrs-8 1000000 1090 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-8 1000000 1097 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-8 1000000 1078 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-8 1000000 1095 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-8 1000000 1096 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-parallel-8 4007268 308.2 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-parallel-8 4016138 299.7 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-parallel-8 4020529 305.9 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-parallel-8 3977829 303.4 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/attrs-parallel-8 3225438 318.5 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/keys-values-8 1179256 994.2 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/keys-values-8 1000000 1002 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/keys-values-8 1216710 993.2 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/keys-values-8 1000000 1013 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/keys-values-8 1000000 1016 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-8 989066 1163 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-8 994116 1163 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-8 1000000 1152 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-8 991675 1165 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-8 965268 1166 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-parallel-8 3955503 303.3 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-parallel-8 3861188 307.8 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-parallel-8 3967752 303.9 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-parallel-8 3955203 302.7 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/WithContext-parallel-8 3948278 301.1 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/Ctx-8 940622 1247 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/Ctx-8 936381 1257 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/Ctx-8 959730 1266 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/Ctx-8 943473 1290 ns/op 0 B/op 0 allocs/op +BenchmarkNopLog/Ctx-8 919414 1259 ns/op 0 B/op 0 allocs/op +PASS +ok golang.org/x/exp/slog 40.566s diff --git a/vendor/golang.org/x/exp/slog/record.go b/vendor/golang.org/x/exp/slog/record.go new file mode 100644 index 0000000..38b3440 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/record.go @@ -0,0 +1,207 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "runtime" + "time" + + "golang.org/x/exp/slices" +) + +const nAttrsInline = 5 + +// A Record holds information about a log event. +// Copies of a Record share state. +// Do not modify a Record after handing out a copy to it. +// Use [Record.Clone] to create a copy with no shared state. +type Record struct { + // The time at which the output method (Log, Info, etc.) was called. + Time time.Time + + // The log message. + Message string + + // The level of the event. + Level Level + + // The program counter at the time the record was constructed, as determined + // by runtime.Callers. If zero, no program counter is available. + // + // The only valid use for this value is as an argument to + // [runtime.CallersFrames]. In particular, it must not be passed to + // [runtime.FuncForPC]. + PC uintptr + + // Allocation optimization: an inline array sized to hold + // the majority of log calls (based on examination of open-source + // code). It holds the start of the list of Attrs. + front [nAttrsInline]Attr + + // The number of Attrs in front. + nFront int + + // The list of Attrs except for those in front. + // Invariants: + // - len(back) > 0 iff nFront == len(front) + // - Unused array elements are zero. Used to detect mistakes. + back []Attr +} + +// NewRecord creates a Record from the given arguments. +// Use [Record.AddAttrs] to add attributes to the Record. +// +// NewRecord is intended for logging APIs that want to support a [Handler] as +// a backend. +func NewRecord(t time.Time, level Level, msg string, pc uintptr) Record { + return Record{ + Time: t, + Message: msg, + Level: level, + PC: pc, + } +} + +// Clone returns a copy of the record with no shared state. +// The original record and the clone can both be modified +// without interfering with each other. +func (r Record) Clone() Record { + r.back = slices.Clip(r.back) // prevent append from mutating shared array + return r +} + +// NumAttrs returns the number of attributes in the Record. +func (r Record) NumAttrs() int { + return r.nFront + len(r.back) +} + +// Attrs calls f on each Attr in the Record. +// Iteration stops if f returns false. +func (r Record) Attrs(f func(Attr) bool) { + for i := 0; i < r.nFront; i++ { + if !f(r.front[i]) { + return + } + } + for _, a := range r.back { + if !f(a) { + return + } + } +} + +// AddAttrs appends the given Attrs to the Record's list of Attrs. +func (r *Record) AddAttrs(attrs ...Attr) { + n := copy(r.front[r.nFront:], attrs) + r.nFront += n + // Check if a copy was modified by slicing past the end + // and seeing if the Attr there is non-zero. + if cap(r.back) > len(r.back) { + end := r.back[:len(r.back)+1][len(r.back)] + if !end.isEmpty() { + panic("copies of a slog.Record were both modified") + } + } + r.back = append(r.back, attrs[n:]...) +} + +// Add converts the args to Attrs as described in [Logger.Log], +// then appends the Attrs to the Record's list of Attrs. +func (r *Record) Add(args ...any) { + var a Attr + for len(args) > 0 { + a, args = argsToAttr(args) + if r.nFront < len(r.front) { + r.front[r.nFront] = a + r.nFront++ + } else { + if r.back == nil { + r.back = make([]Attr, 0, countAttrs(args)) + } + r.back = append(r.back, a) + } + } + +} + +// countAttrs returns the number of Attrs that would be created from args. +func countAttrs(args []any) int { + n := 0 + for i := 0; i < len(args); i++ { + n++ + if _, ok := args[i].(string); ok { + i++ + } + } + return n +} + +const badKey = "!BADKEY" + +// argsToAttr turns a prefix of the nonempty args slice into an Attr +// and returns the unconsumed portion of the slice. +// If args[0] is an Attr, it returns it. +// If args[0] is a string, it treats the first two elements as +// a key-value pair. +// Otherwise, it treats args[0] as a value with a missing key. +func argsToAttr(args []any) (Attr, []any) { + switch x := args[0].(type) { + case string: + if len(args) == 1 { + return String(badKey, x), nil + } + return Any(x, args[1]), args[2:] + + case Attr: + return x, args[1:] + + default: + return Any(badKey, x), args[1:] + } +} + +// Source describes the location of a line of source code. +type Source struct { + // Function is the package path-qualified function name containing the + // source line. If non-empty, this string uniquely identifies a single + // function in the program. This may be the empty string if not known. + Function string `json:"function"` + // File and Line are the file name and line number (1-based) of the source + // line. These may be the empty string and zero, respectively, if not known. + File string `json:"file"` + Line int `json:"line"` +} + +// attrs returns the non-zero fields of s as a slice of attrs. +// It is similar to a LogValue method, but we don't want Source +// to implement LogValuer because it would be resolved before +// the ReplaceAttr function was called. +func (s *Source) group() Value { + var as []Attr + if s.Function != "" { + as = append(as, String("function", s.Function)) + } + if s.File != "" { + as = append(as, String("file", s.File)) + } + if s.Line != 0 { + as = append(as, Int("line", s.Line)) + } + return GroupValue(as...) +} + +// source returns a Source for the log event. +// If the Record was created without the necessary information, +// or if the location is unavailable, it returns a non-nil *Source +// with zero fields. +func (r Record) source() *Source { + fs := runtime.CallersFrames([]uintptr{r.PC}) + f, _ := fs.Next() + return &Source{ + Function: f.Function, + File: f.File, + Line: f.Line, + } +} diff --git a/vendor/golang.org/x/exp/slog/text_handler.go b/vendor/golang.org/x/exp/slog/text_handler.go new file mode 100644 index 0000000..75b66b7 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/text_handler.go @@ -0,0 +1,161 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "context" + "encoding" + "fmt" + "io" + "reflect" + "strconv" + "unicode" + "unicode/utf8" +) + +// TextHandler is a Handler that writes Records to an io.Writer as a +// sequence of key=value pairs separated by spaces and followed by a newline. +type TextHandler struct { + *commonHandler +} + +// NewTextHandler creates a TextHandler that writes to w, +// using the given options. +// If opts is nil, the default options are used. +func NewTextHandler(w io.Writer, opts *HandlerOptions) *TextHandler { + if opts == nil { + opts = &HandlerOptions{} + } + return &TextHandler{ + &commonHandler{ + json: false, + w: w, + opts: *opts, + }, + } +} + +// Enabled reports whether the handler handles records at the given level. +// The handler ignores records whose level is lower. +func (h *TextHandler) Enabled(_ context.Context, level Level) bool { + return h.commonHandler.enabled(level) +} + +// WithAttrs returns a new TextHandler whose attributes consists +// of h's attributes followed by attrs. +func (h *TextHandler) WithAttrs(attrs []Attr) Handler { + return &TextHandler{commonHandler: h.commonHandler.withAttrs(attrs)} +} + +func (h *TextHandler) WithGroup(name string) Handler { + return &TextHandler{commonHandler: h.commonHandler.withGroup(name)} +} + +// Handle formats its argument Record as a single line of space-separated +// key=value items. +// +// If the Record's time is zero, the time is omitted. +// Otherwise, the key is "time" +// and the value is output in RFC3339 format with millisecond precision. +// +// If the Record's level is zero, the level is omitted. +// Otherwise, the key is "level" +// and the value of [Level.String] is output. +// +// If the AddSource option is set and source information is available, +// the key is "source" and the value is output as FILE:LINE. +// +// The message's key is "msg". +// +// To modify these or other attributes, or remove them from the output, use +// [HandlerOptions.ReplaceAttr]. +// +// If a value implements [encoding.TextMarshaler], the result of MarshalText is +// written. Otherwise, the result of fmt.Sprint is written. +// +// Keys and values are quoted with [strconv.Quote] if they contain Unicode space +// characters, non-printing characters, '"' or '='. +// +// Keys inside groups consist of components (keys or group names) separated by +// dots. No further escaping is performed. +// Thus there is no way to determine from the key "a.b.c" whether there +// are two groups "a" and "b" and a key "c", or a single group "a.b" and a key "c", +// or single group "a" and a key "b.c". +// If it is necessary to reconstruct the group structure of a key +// even in the presence of dots inside components, use +// [HandlerOptions.ReplaceAttr] to encode that information in the key. +// +// Each call to Handle results in a single serialized call to +// io.Writer.Write. +func (h *TextHandler) Handle(_ context.Context, r Record) error { + return h.commonHandler.handle(r) +} + +func appendTextValue(s *handleState, v Value) error { + switch v.Kind() { + case KindString: + s.appendString(v.str()) + case KindTime: + s.appendTime(v.time()) + case KindAny: + if tm, ok := v.any.(encoding.TextMarshaler); ok { + data, err := tm.MarshalText() + if err != nil { + return err + } + // TODO: avoid the conversion to string. + s.appendString(string(data)) + return nil + } + if bs, ok := byteSlice(v.any); ok { + // As of Go 1.19, this only allocates for strings longer than 32 bytes. + s.buf.WriteString(strconv.Quote(string(bs))) + return nil + } + s.appendString(fmt.Sprintf("%+v", v.Any())) + default: + *s.buf = v.append(*s.buf) + } + return nil +} + +// byteSlice returns its argument as a []byte if the argument's +// underlying type is []byte, along with a second return value of true. +// Otherwise it returns nil, false. +func byteSlice(a any) ([]byte, bool) { + if bs, ok := a.([]byte); ok { + return bs, true + } + // Like Printf's %s, we allow both the slice type and the byte element type to be named. + t := reflect.TypeOf(a) + if t != nil && t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 { + return reflect.ValueOf(a).Bytes(), true + } + return nil, false +} + +func needsQuoting(s string) bool { + if len(s) == 0 { + return true + } + for i := 0; i < len(s); { + b := s[i] + if b < utf8.RuneSelf { + // Quote anything except a backslash that would need quoting in a + // JSON string, as well as space and '=' + if b != '\\' && (b == ' ' || b == '=' || !safeSet[b]) { + return true + } + i++ + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError || unicode.IsSpace(r) || !unicode.IsPrint(r) { + return true + } + i += size + } + return false +} diff --git a/vendor/golang.org/x/exp/slog/value.go b/vendor/golang.org/x/exp/slog/value.go new file mode 100644 index 0000000..3550c46 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/value.go @@ -0,0 +1,456 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "fmt" + "math" + "runtime" + "strconv" + "strings" + "time" + "unsafe" + + "golang.org/x/exp/slices" +) + +// A Value can represent any Go value, but unlike type any, +// it can represent most small values without an allocation. +// The zero Value corresponds to nil. +type Value struct { + _ [0]func() // disallow == + // num holds the value for Kinds Int64, Uint64, Float64, Bool and Duration, + // the string length for KindString, and nanoseconds since the epoch for KindTime. + num uint64 + // If any is of type Kind, then the value is in num as described above. + // If any is of type *time.Location, then the Kind is Time and time.Time value + // can be constructed from the Unix nanos in num and the location (monotonic time + // is not preserved). + // If any is of type stringptr, then the Kind is String and the string value + // consists of the length in num and the pointer in any. + // Otherwise, the Kind is Any and any is the value. + // (This implies that Attrs cannot store values of type Kind, *time.Location + // or stringptr.) + any any +} + +// Kind is the kind of a Value. +type Kind int + +// The following list is sorted alphabetically, but it's also important that +// KindAny is 0 so that a zero Value represents nil. + +const ( + KindAny Kind = iota + KindBool + KindDuration + KindFloat64 + KindInt64 + KindString + KindTime + KindUint64 + KindGroup + KindLogValuer +) + +var kindStrings = []string{ + "Any", + "Bool", + "Duration", + "Float64", + "Int64", + "String", + "Time", + "Uint64", + "Group", + "LogValuer", +} + +func (k Kind) String() string { + if k >= 0 && int(k) < len(kindStrings) { + return kindStrings[k] + } + return "" +} + +// Unexported version of Kind, just so we can store Kinds in Values. +// (No user-provided value has this type.) +type kind Kind + +// Kind returns v's Kind. +func (v Value) Kind() Kind { + switch x := v.any.(type) { + case Kind: + return x + case stringptr: + return KindString + case timeLocation: + return KindTime + case groupptr: + return KindGroup + case LogValuer: + return KindLogValuer + case kind: // a kind is just a wrapper for a Kind + return KindAny + default: + return KindAny + } +} + +//////////////// Constructors + +// IntValue returns a Value for an int. +func IntValue(v int) Value { + return Int64Value(int64(v)) +} + +// Int64Value returns a Value for an int64. +func Int64Value(v int64) Value { + return Value{num: uint64(v), any: KindInt64} +} + +// Uint64Value returns a Value for a uint64. +func Uint64Value(v uint64) Value { + return Value{num: v, any: KindUint64} +} + +// Float64Value returns a Value for a floating-point number. +func Float64Value(v float64) Value { + return Value{num: math.Float64bits(v), any: KindFloat64} +} + +// BoolValue returns a Value for a bool. +func BoolValue(v bool) Value { + u := uint64(0) + if v { + u = 1 + } + return Value{num: u, any: KindBool} +} + +// Unexported version of *time.Location, just so we can store *time.Locations in +// Values. (No user-provided value has this type.) +type timeLocation *time.Location + +// TimeValue returns a Value for a time.Time. +// It discards the monotonic portion. +func TimeValue(v time.Time) Value { + if v.IsZero() { + // UnixNano on the zero time is undefined, so represent the zero time + // with a nil *time.Location instead. time.Time.Location method never + // returns nil, so a Value with any == timeLocation(nil) cannot be + // mistaken for any other Value, time.Time or otherwise. + return Value{any: timeLocation(nil)} + } + return Value{num: uint64(v.UnixNano()), any: timeLocation(v.Location())} +} + +// DurationValue returns a Value for a time.Duration. +func DurationValue(v time.Duration) Value { + return Value{num: uint64(v.Nanoseconds()), any: KindDuration} +} + +// AnyValue returns a Value for the supplied value. +// +// If the supplied value is of type Value, it is returned +// unmodified. +// +// Given a value of one of Go's predeclared string, bool, or +// (non-complex) numeric types, AnyValue returns a Value of kind +// String, Bool, Uint64, Int64, or Float64. The width of the +// original numeric type is not preserved. +// +// Given a time.Time or time.Duration value, AnyValue returns a Value of kind +// KindTime or KindDuration. The monotonic time is not preserved. +// +// For nil, or values of all other types, including named types whose +// underlying type is numeric, AnyValue returns a value of kind KindAny. +func AnyValue(v any) Value { + switch v := v.(type) { + case string: + return StringValue(v) + case int: + return Int64Value(int64(v)) + case uint: + return Uint64Value(uint64(v)) + case int64: + return Int64Value(v) + case uint64: + return Uint64Value(v) + case bool: + return BoolValue(v) + case time.Duration: + return DurationValue(v) + case time.Time: + return TimeValue(v) + case uint8: + return Uint64Value(uint64(v)) + case uint16: + return Uint64Value(uint64(v)) + case uint32: + return Uint64Value(uint64(v)) + case uintptr: + return Uint64Value(uint64(v)) + case int8: + return Int64Value(int64(v)) + case int16: + return Int64Value(int64(v)) + case int32: + return Int64Value(int64(v)) + case float64: + return Float64Value(v) + case float32: + return Float64Value(float64(v)) + case []Attr: + return GroupValue(v...) + case Kind: + return Value{any: kind(v)} + case Value: + return v + default: + return Value{any: v} + } +} + +//////////////// Accessors + +// Any returns v's value as an any. +func (v Value) Any() any { + switch v.Kind() { + case KindAny: + if k, ok := v.any.(kind); ok { + return Kind(k) + } + return v.any + case KindLogValuer: + return v.any + case KindGroup: + return v.group() + case KindInt64: + return int64(v.num) + case KindUint64: + return v.num + case KindFloat64: + return v.float() + case KindString: + return v.str() + case KindBool: + return v.bool() + case KindDuration: + return v.duration() + case KindTime: + return v.time() + default: + panic(fmt.Sprintf("bad kind: %s", v.Kind())) + } +} + +// Int64 returns v's value as an int64. It panics +// if v is not a signed integer. +func (v Value) Int64() int64 { + if g, w := v.Kind(), KindInt64; g != w { + panic(fmt.Sprintf("Value kind is %s, not %s", g, w)) + } + return int64(v.num) +} + +// Uint64 returns v's value as a uint64. It panics +// if v is not an unsigned integer. +func (v Value) Uint64() uint64 { + if g, w := v.Kind(), KindUint64; g != w { + panic(fmt.Sprintf("Value kind is %s, not %s", g, w)) + } + return v.num +} + +// Bool returns v's value as a bool. It panics +// if v is not a bool. +func (v Value) Bool() bool { + if g, w := v.Kind(), KindBool; g != w { + panic(fmt.Sprintf("Value kind is %s, not %s", g, w)) + } + return v.bool() +} + +func (v Value) bool() bool { + return v.num == 1 +} + +// Duration returns v's value as a time.Duration. It panics +// if v is not a time.Duration. +func (v Value) Duration() time.Duration { + if g, w := v.Kind(), KindDuration; g != w { + panic(fmt.Sprintf("Value kind is %s, not %s", g, w)) + } + + return v.duration() +} + +func (v Value) duration() time.Duration { + return time.Duration(int64(v.num)) +} + +// Float64 returns v's value as a float64. It panics +// if v is not a float64. +func (v Value) Float64() float64 { + if g, w := v.Kind(), KindFloat64; g != w { + panic(fmt.Sprintf("Value kind is %s, not %s", g, w)) + } + + return v.float() +} + +func (v Value) float() float64 { + return math.Float64frombits(v.num) +} + +// Time returns v's value as a time.Time. It panics +// if v is not a time.Time. +func (v Value) Time() time.Time { + if g, w := v.Kind(), KindTime; g != w { + panic(fmt.Sprintf("Value kind is %s, not %s", g, w)) + } + return v.time() +} + +func (v Value) time() time.Time { + loc := v.any.(timeLocation) + if loc == nil { + return time.Time{} + } + return time.Unix(0, int64(v.num)).In(loc) +} + +// LogValuer returns v's value as a LogValuer. It panics +// if v is not a LogValuer. +func (v Value) LogValuer() LogValuer { + return v.any.(LogValuer) +} + +// Group returns v's value as a []Attr. +// It panics if v's Kind is not KindGroup. +func (v Value) Group() []Attr { + if sp, ok := v.any.(groupptr); ok { + return unsafe.Slice((*Attr)(sp), v.num) + } + panic("Group: bad kind") +} + +func (v Value) group() []Attr { + return unsafe.Slice((*Attr)(v.any.(groupptr)), v.num) +} + +//////////////// Other + +// Equal reports whether v and w represent the same Go value. +func (v Value) Equal(w Value) bool { + k1 := v.Kind() + k2 := w.Kind() + if k1 != k2 { + return false + } + switch k1 { + case KindInt64, KindUint64, KindBool, KindDuration: + return v.num == w.num + case KindString: + return v.str() == w.str() + case KindFloat64: + return v.float() == w.float() + case KindTime: + return v.time().Equal(w.time()) + case KindAny, KindLogValuer: + return v.any == w.any // may panic if non-comparable + case KindGroup: + return slices.EqualFunc(v.group(), w.group(), Attr.Equal) + default: + panic(fmt.Sprintf("bad kind: %s", k1)) + } +} + +// append appends a text representation of v to dst. +// v is formatted as with fmt.Sprint. +func (v Value) append(dst []byte) []byte { + switch v.Kind() { + case KindString: + return append(dst, v.str()...) + case KindInt64: + return strconv.AppendInt(dst, int64(v.num), 10) + case KindUint64: + return strconv.AppendUint(dst, v.num, 10) + case KindFloat64: + return strconv.AppendFloat(dst, v.float(), 'g', -1, 64) + case KindBool: + return strconv.AppendBool(dst, v.bool()) + case KindDuration: + return append(dst, v.duration().String()...) + case KindTime: + return append(dst, v.time().String()...) + case KindGroup: + return fmt.Append(dst, v.group()) + case KindAny, KindLogValuer: + return fmt.Append(dst, v.any) + default: + panic(fmt.Sprintf("bad kind: %s", v.Kind())) + } +} + +// A LogValuer is any Go value that can convert itself into a Value for logging. +// +// This mechanism may be used to defer expensive operations until they are +// needed, or to expand a single value into a sequence of components. +type LogValuer interface { + LogValue() Value +} + +const maxLogValues = 100 + +// Resolve repeatedly calls LogValue on v while it implements LogValuer, +// and returns the result. +// If v resolves to a group, the group's attributes' values are not recursively +// resolved. +// If the number of LogValue calls exceeds a threshold, a Value containing an +// error is returned. +// Resolve's return value is guaranteed not to be of Kind KindLogValuer. +func (v Value) Resolve() (rv Value) { + orig := v + defer func() { + if r := recover(); r != nil { + rv = AnyValue(fmt.Errorf("LogValue panicked\n%s", stack(3, 5))) + } + }() + + for i := 0; i < maxLogValues; i++ { + if v.Kind() != KindLogValuer { + return v + } + v = v.LogValuer().LogValue() + } + err := fmt.Errorf("LogValue called too many times on Value of type %T", orig.Any()) + return AnyValue(err) +} + +func stack(skip, nFrames int) string { + pcs := make([]uintptr, nFrames+1) + n := runtime.Callers(skip+1, pcs) + if n == 0 { + return "(no stack)" + } + frames := runtime.CallersFrames(pcs[:n]) + var b strings.Builder + i := 0 + for { + frame, more := frames.Next() + fmt.Fprintf(&b, "called from %s (%s:%d)\n", frame.Function, frame.File, frame.Line) + if !more { + break + } + i++ + if i >= nFrames { + fmt.Fprintf(&b, "(rest of stack elided)\n") + break + } + } + return b.String() +} diff --git a/vendor/golang.org/x/exp/slog/value_119.go b/vendor/golang.org/x/exp/slog/value_119.go new file mode 100644 index 0000000..29b0d73 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/value_119.go @@ -0,0 +1,53 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 && !go1.20 + +package slog + +import ( + "reflect" + "unsafe" +) + +type ( + stringptr unsafe.Pointer // used in Value.any when the Value is a string + groupptr unsafe.Pointer // used in Value.any when the Value is a []Attr +) + +// StringValue returns a new Value for a string. +func StringValue(value string) Value { + hdr := (*reflect.StringHeader)(unsafe.Pointer(&value)) + return Value{num: uint64(hdr.Len), any: stringptr(hdr.Data)} +} + +func (v Value) str() string { + var s string + hdr := (*reflect.StringHeader)(unsafe.Pointer(&s)) + hdr.Data = uintptr(v.any.(stringptr)) + hdr.Len = int(v.num) + return s +} + +// String returns Value's value as a string, formatted like fmt.Sprint. Unlike +// the methods Int64, Float64, and so on, which panic if v is of the +// wrong kind, String never panics. +func (v Value) String() string { + if sp, ok := v.any.(stringptr); ok { + // Inlining this code makes a huge difference. + var s string + hdr := (*reflect.StringHeader)(unsafe.Pointer(&s)) + hdr.Data = uintptr(sp) + hdr.Len = int(v.num) + return s + } + return string(v.append(nil)) +} + +// GroupValue returns a new Value for a list of Attrs. +// The caller must not subsequently mutate the argument slice. +func GroupValue(as ...Attr) Value { + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&as)) + return Value{num: uint64(hdr.Len), any: groupptr(hdr.Data)} +} diff --git a/vendor/golang.org/x/exp/slog/value_120.go b/vendor/golang.org/x/exp/slog/value_120.go new file mode 100644 index 0000000..f7d4c09 --- /dev/null +++ b/vendor/golang.org/x/exp/slog/value_120.go @@ -0,0 +1,39 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.20 + +package slog + +import "unsafe" + +type ( + stringptr *byte // used in Value.any when the Value is a string + groupptr *Attr // used in Value.any when the Value is a []Attr +) + +// StringValue returns a new Value for a string. +func StringValue(value string) Value { + return Value{num: uint64(len(value)), any: stringptr(unsafe.StringData(value))} +} + +// GroupValue returns a new Value for a list of Attrs. +// The caller must not subsequently mutate the argument slice. +func GroupValue(as ...Attr) Value { + return Value{num: uint64(len(as)), any: groupptr(unsafe.SliceData(as))} +} + +// String returns Value's value as a string, formatted like fmt.Sprint. Unlike +// the methods Int64, Float64, and so on, which panic if v is of the +// wrong kind, String never panics. +func (v Value) String() string { + if sp, ok := v.any.(stringptr); ok { + return unsafe.String(sp, v.num) + } + return string(v.append(nil)) +} + +func (v Value) str() string { + return unsafe.String(v.any.(stringptr), v.num) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 745840e..d0ae9b7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -76,6 +76,9 @@ golang.org/x/crypto/pbkdf2 golang.org/x/exp/constraints golang.org/x/exp/maps golang.org/x/exp/slices +golang.org/x/exp/slog +golang.org/x/exp/slog/internal +golang.org/x/exp/slog/internal/buffer # golang.org/x/mod v0.14.0 ## explicit; go 1.18 golang.org/x/mod/internal/lazyregexp diff --git a/webaccount/account.go b/webaccount/account.go index 88084b2..6782747 100644 --- a/webaccount/account.go +++ b/webaccount/account.go @@ -20,6 +20,8 @@ import ( _ "embed" + "golang.org/x/exp/slog" + "github.com/mjl-/sherpa" "github.com/mjl-/sherpadoc" "github.com/mjl-/sherpaprom" @@ -37,7 +39,7 @@ func init() { mox.LimitersInit() } -var xlog = mlog.New("webaccount") +var pkglog = mlog.New("webaccount", nil) //go:embed accountapi.json var accountapiJSON []byte @@ -52,7 +54,7 @@ var accountSherpaHandler http.Handler func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) { err := json.Unmarshal(buf, &doc) if err != nil { - xlog.Fatalx("parsing webaccount api docs", err, mlog.Field("api", api)) + pkglog.Fatalx("parsing webaccount api docs", err, slog.String("api", api)) } return doc } @@ -60,12 +62,12 @@ func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) { func init() { collector, err := sherpaprom.NewCollector("moxaccount", nil) if err != nil { - xlog.Fatalx("creating sherpa prometheus collector", err) + pkglog.Fatalx("creating sherpa prometheus collector", err) } accountSherpaHandler, err = sherpa.NewHandler("/api/", moxvar.Version, Account{}, &accountDoc, &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none"}) if err != nil { - xlog.Fatalx("sherpa handler", err) + pkglog.Fatalx("sherpa handler", err) } } @@ -75,7 +77,7 @@ func xcheckf(ctx context.Context, err error, format string, args ...any) { } msg := fmt.Sprintf(format, args...) errmsg := fmt.Sprintf("%s: %s", msg, err) - xlog.WithContext(ctx).Errorx(msg, err) + pkglog.WithContext(ctx).Errorx(msg, err) panic(&sherpa.Error{Code: "server:error", Message: errmsg}) } @@ -85,7 +87,7 @@ func xcheckuserf(ctx context.Context, err error, format string, args ...any) { } msg := fmt.Sprintf(format, args...) errmsg := fmt.Sprintf("%s: %s", msg, err) - xlog.WithContext(ctx).Errorx(msg, err) + pkglog.WithContext(ctx).Errorx(msg, err) panic(&sherpa.Error{Code: "user:error", Message: errmsg}) } @@ -96,7 +98,7 @@ type Account struct{} // CheckAuth checks http basic auth, returns login address and account name if // valid, and writes http response and returns empty string otherwise. -func CheckAuth(ctx context.Context, log *mlog.Log, kind string, w http.ResponseWriter, r *http.Request) (address, account string) { +func CheckAuth(ctx context.Context, log mlog.Log, kind string, w http.ResponseWriter, r *http.Request) (address, account string) { authResult := "error" start := time.Now() var addr *net.TCPAddr @@ -111,7 +113,7 @@ func CheckAuth(ctx context.Context, log *mlog.Log, kind string, w http.ResponseW var remoteIP net.IP addr, err = net.ResolveTCPAddr("tcp", r.RemoteAddr) if err != nil { - log.Errorx("parsing remote address", err, mlog.Field("addr", r.RemoteAddr)) + log.Errorx("parsing remote address", err, slog.Any("addr", r.RemoteAddr)) } else if addr != nil { remoteIP = addr.IP } @@ -127,10 +129,10 @@ func CheckAuth(ctx context.Context, log *mlog.Log, kind string, w http.ResponseW log.Debugx("parsing base64", err) } else if t := strings.SplitN(string(authBuf), ":", 2); len(t) != 2 { log.Debug("bad user:pass form") - } else if acc, err := store.OpenEmailAuth(t[0], t[1]); err != nil { + } else if acc, err := store.OpenEmailAuth(log, t[0], t[1]); err != nil { if errors.Is(err, store.ErrUnknownCredentials) { authResult = "badcreds" - log.Info("failed authentication attempt", mlog.Field("username", t[0]), mlog.Field("remote", remoteIP)) + log.Info("failed authentication attempt", slog.String("username", t[0]), slog.Any("remote", remoteIP)) } log.Errorx("open account", err) } else { @@ -148,7 +150,7 @@ func CheckAuth(ctx context.Context, log *mlog.Log, kind string, w http.ResponseW func Handle(w http.ResponseWriter, r *http.Request) { ctx := context.WithValue(r.Context(), mlog.CidKey, mox.Cid()) - log := xlog.WithContext(ctx).Fields(mlog.Field("userauth", "")) + log := pkglog.WithContext(ctx).With(slog.String("userauth", "")) // Without authentication. The token is unguessable. if r.URL.Path == "/importprogress" { @@ -213,8 +215,8 @@ func Handle(w http.ResponseWriter, r *http.Request) { return } - if lw, ok := w.(interface{ AddField(p mlog.Pair) }); ok { - lw.AddField(mlog.Field("authaccount", accName)) + if lw, ok := w.(interface{ AddAttr(a slog.Attr) }); ok { + lw.AddAttr(slog.String("authaccount", accName)) } switch r.URL.Path { @@ -239,7 +241,7 @@ func Handle(w http.ResponseWriter, r *http.Request) { maildir := strings.Contains(r.URL.Path, "maildir") tgz := strings.Contains(r.URL.Path, ".tgz") - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(log, accName) if err != nil { log.Errorx("open account for export", err) http.Error(w, "500 - internal server error", http.StatusInternalServerError) @@ -336,17 +338,18 @@ var authCtxKey ctxKey = "account" // Sessions are not interrupted, and will keep working. New login attempts must use the new password. // Password must be at least 8 characters. func (Account) SetPassword(ctx context.Context, password string) { + log := pkglog.WithContext(ctx) if len(password) < 8 { panic(&sherpa.Error{Code: "user:error", Message: "password must be at least 8 characters"}) } accountName := ctx.Value(authCtxKey).(string) - acc, err := store.OpenAccount(accountName) + acc, err := store.OpenAccount(log, accountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() - xlog.Check(err, "closing account") + log.Check(err, "closing account") }() - err = acc.SetPassword(password) + err = acc.SetPassword(log, password) xcheckf(ctx, err, "setting password") } diff --git a/webaccount/account_test.go b/webaccount/account_test.go index 688b474..0f32849 100644 --- a/webaccount/account_test.go +++ b/webaccount/account_test.go @@ -41,7 +41,8 @@ func TestAccount(t *testing.T) { mox.ConfigStaticPath = filepath.FromSlash("../testdata/httpaccount/mox.conf") mox.ConfigDynamicPath = filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf") mox.MustLoadConfig(true, false) - acc, err := store.OpenAccount("mjl") + log := mlog.New("webaccount", nil) + acc, err := store.OpenAccount(log, "mjl") tcheck(t, err, "open account") defer func() { err = acc.Close() @@ -49,8 +50,6 @@ func TestAccount(t *testing.T) { }() defer store.Switchboard()() - log := mlog.New("store") - test := func(userpass string, expect string) { t.Helper() diff --git a/webaccount/import.go b/webaccount/import.go index 2f2e14f..28c6a4e 100644 --- a/webaccount/import.go +++ b/webaccount/import.go @@ -21,6 +21,7 @@ import ( "time" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "golang.org/x/text/unicode/norm" "github.com/mjl-/bstore" @@ -64,10 +65,10 @@ var importers = struct { // ImportManage should be run as a goroutine, it manages imports of mboxes/maildirs, propagating progress over SSE connections. func ImportManage() { - log := mlog.New("httpimport") + log := mlog.New("httpimport", nil) defer func() { if x := recover(); x != nil { - log.Error("import manage panic", mlog.Field("err", x)) + log.Error("import manage panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Importmanage) } @@ -94,7 +95,7 @@ func ImportManage() { sendEvent := func(kind string, v any) { buf, err := json.Marshal(v) if err != nil { - log.Errorx("marshal event", err, mlog.Field("kind", kind), mlog.Field("event", v)) + log.Errorx("marshal event", err, slog.String("kind", kind), slog.Any("event", v)) return } ssemsg := fmt.Sprintf("event: %s\ndata: %s\n\n", kind, buf) @@ -199,7 +200,7 @@ type importStep struct { // importStart prepare the import and launches the goroutine to actually import. // importStart is responsible for closing f and removing f. -func importStart(log *mlog.Log, accName string, f *os.File, skipMailboxPrefix string) (string, error) { +func importStart(log mlog.Log, accName string, f *os.File, skipMailboxPrefix string) (string, error) { defer func() { if f != nil { store.CloseRemoveTempFile(log, f, "upload for import") @@ -249,7 +250,7 @@ func importStart(log *mlog.Log, accName string, f *os.File, skipMailboxPrefix st tr = tar.NewReader(gzr) } - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(log, accName) if err != nil { return "", fmt.Errorf("open acount: %v", err) } @@ -276,7 +277,7 @@ func importStart(log *mlog.Log, accName string, f *os.File, skipMailboxPrefix st // importMessages imports the messages from zip/tgz file f. // importMessages is responsible for unlocking and closing acc, and closing tx and f. -func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store.Account, tx *bstore.Tx, zr *zip.Reader, tr *tar.Reader, f *os.File, skipMailboxPrefix string) { +func importMessages(ctx context.Context, log mlog.Log, token string, acc *store.Account, tx *bstore.Tx, zr *zip.Reader, tr *tar.Reader, f *os.File, skipMailboxPrefix string) { // If a fatal processing error occurs, we panic with this type. type importError struct{ Err error } @@ -289,7 +290,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store sendEvent := func(kind string, v any) { buf, err := json.Marshal(v) if err != nil { - log.Errorx("marshal event", err, mlog.Field("kind", kind), mlog.Field("event", v)) + log.Errorx("marshal event", err, slog.String("kind", kind), slog.Any("event", v)) return } ssemsg := fmt.Sprintf("event: %s\ndata: %s\n\n", kind, buf) @@ -317,7 +318,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store for _, id := range deliveredIDs { p := acc.MessagePath(id) err := os.Remove(p) - log.Check(err, "closing message file after import error", mlog.Field("path", p)) + log.Check(err, "closing message file after import error", slog.String("path", p)) } if tx != nil { err := tx.Rollback() @@ -338,7 +339,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store problemf("%s (aborting)", err.Err) sendEvent("aborted", importAborted{}) } else { - log.Error("import panic", mlog.Field("err", x)) + log.Error("import panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Importmessages) } @@ -511,7 +512,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store } // Parse message and store parsed information for later fast retrieval. - p, err := message.EnsurePart(log, false, f, m.Size) + p, err := message.EnsurePart(log.Logger, false, f, m.Size) if err != nil { problemf("parsing message %s: %s (continuing)", pos, err) } @@ -562,7 +563,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store } mb := xensureMailbox(mailbox) - mr := store.NewMboxReader(store.CreateMessageTemp, filename, r, log) + mr := store.NewMboxReader(log, store.CreateMessageTemp, filename, r) for { m, mf, pos, err := mr.Next() if err == io.EOF { @@ -582,7 +583,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store } mb := xensureMailbox(mailbox) - f, err := store.CreateMessageTemp("import") + f, err := store.CreateMessageTemp(log, "import") ximportcheckf(err, "creating temp message") defer func() { if f != nil { @@ -707,7 +708,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store mailbox := path.Dir(name) dovecotKeywords := map[rune]string{} words, err := store.ParseDovecotKeywordsFlags(r, log) - log.Check(err, "parsing dovecot keywords for mailbox", mlog.Field("mailbox", mailbox)) + log.Check(err, "parsing dovecot keywords for mailbox", slog.String("mailbox", mailbox)) for i, kw := range words { dovecotKeywords['a'+rune(i)] = kw } @@ -801,7 +802,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store for _, count := range messages { total += count } - log.Debug("messages imported", mlog.Field("total", total)) + log.Debug("messages imported", slog.Int("total", total)) // Send final update for count of last-imported mailbox. if prevMailbox != "" { diff --git a/webadmin/admin.go b/webadmin/admin.go index fc81c89..5cd21ae 100644 --- a/webadmin/admin.go +++ b/webadmin/admin.go @@ -34,6 +34,7 @@ import ( "golang.org/x/crypto/bcrypt" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/mjl-/adns" @@ -64,7 +65,7 @@ import ( "github.com/mjl-/mox/tlsrptdb" ) -var xlog = mlog.New("webadmin") +var pkglog = mlog.New("webadmin", nil) //go:embed adminapi.json var adminapiJSON []byte @@ -79,7 +80,7 @@ var adminSherpaHandler http.Handler func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) { err := json.Unmarshal(buf, &doc) if err != nil { - xlog.Fatalx("parsing webadmin api docs", err, mlog.Field("api", api)) + pkglog.Fatalx("parsing webadmin api docs", err, slog.String("api", api)) } return doc } @@ -87,12 +88,12 @@ func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) { func init() { collector, err := sherpaprom.NewCollector("moxadmin", nil) if err != nil { - xlog.Fatalx("creating sherpa prometheus collector", err) + pkglog.Fatalx("creating sherpa prometheus collector", err) } adminSherpaHandler, err = sherpa.NewHandler("/api/", moxvar.Version, Admin{}, &adminDoc, &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none"}) if err != nil { - xlog.Fatalx("sherpa handler", err) + pkglog.Fatalx("sherpa handler", err) } } @@ -125,7 +126,7 @@ func ManageAuthCache() { // matches the authorization header "authHdr". we don't care about any username. // on (auth) failure, a http response is sent and false returned. func checkAdminAuth(ctx context.Context, passwordfile string, w http.ResponseWriter, r *http.Request) bool { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) respondAuthFail := func() bool { // note: browsers don't display the realm to prevent users getting confused by malicious realm messages. @@ -148,7 +149,7 @@ func checkAdminAuth(ctx context.Context, passwordfile string, w http.ResponseWri var remoteIP net.IP addr, err = net.ResolveTCPAddr("tcp", r.RemoteAddr) if err != nil { - log.Errorx("parsing remote address", err, mlog.Field("addr", r.RemoteAddr)) + log.Errorx("parsing remote address", err, slog.Any("addr", r.RemoteAddr)) } else if addr != nil { remoteIP = addr.IP } @@ -164,7 +165,7 @@ func checkAdminAuth(ctx context.Context, passwordfile string, w http.ResponseWri } buf, err := os.ReadFile(passwordfile) if err != nil { - log.Errorx("reading admin password file", err, mlog.Field("path", passwordfile)) + log.Errorx("reading admin password file", err, slog.String("path", passwordfile)) return respondAuthFail() } passwordhash := strings.TrimSpace(string(buf)) @@ -180,12 +181,12 @@ func checkAdminAuth(ctx context.Context, passwordfile string, w http.ResponseWri } t := strings.SplitN(string(auth), ":", 2) if len(t) != 2 || len(t[1]) < 8 { - log.Info("failed authentication attempt", mlog.Field("username", "admin"), mlog.Field("remote", remoteIP)) + log.Info("failed authentication attempt", slog.String("username", "admin"), slog.Any("remote", remoteIP)) return respondAuthFail() } if err := bcrypt.CompareHashAndPassword([]byte(passwordhash), []byte(t[1])); err != nil { authResult = "badcreds" - log.Info("failed authentication attempt", mlog.Field("username", "admin"), mlog.Field("remote", remoteIP)) + log.Info("failed authentication attempt", slog.String("username", "admin"), slog.Any("remote", remoteIP)) return respondAuthFail() } authCache.lastSuccessHash = passwordhash @@ -201,8 +202,8 @@ func Handle(w http.ResponseWriter, r *http.Request) { return } - if lw, ok := w.(interface{ AddField(f mlog.Pair) }); ok { - lw.AddField(mlog.Field("authadmin", true)) + if lw, ok := w.(interface{ AddAttr(a slog.Attr) }); ok { + lw.AddAttr(slog.Bool("authadmin", true)) } if r.Method == "GET" && r.URL.Path == "/" { @@ -228,7 +229,7 @@ func xcheckf(ctx context.Context, err error, format string, args ...any) { } msg := fmt.Sprintf(format, args...) errmsg := fmt.Sprintf("%s: %s", msg, err) - xlog.WithContext(ctx).Errorx(msg, err) + pkglog.WithContext(ctx).Errorx(msg, err) panic(&sherpa.Error{Code: "server:error", Message: errmsg}) } @@ -238,7 +239,7 @@ func xcheckuserf(ctx context.Context, err error, format string, args ...any) { } msg := fmt.Sprintf(format, args...) errmsg := fmt.Sprintf("%s: %s", msg, err) - xlog.WithContext(ctx).Errorx(msg, err) + pkglog.WithContext(ctx).Errorx(msg, err) panic(&sherpa.Error{Code: "user:error", Message: errmsg}) } @@ -379,8 +380,7 @@ func logPanic(ctx context.Context) { if x == nil { return } - log := xlog.WithContext(ctx) - log.Error("recover from panic", mlog.Field("panic", x)) + pkglog.WithContext(ctx).Error("recover from panic", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Webadmin) } @@ -404,7 +404,7 @@ func xsendingIPs(ctx context.Context) []net.IP { func (Admin) CheckDomain(ctx context.Context, domainName string) (r CheckResult) { // todo future: should run these checks without a DNS cache so recent changes are picked up. - resolver := dns.StrictResolver{Pkg: "check"} + resolver := dns.StrictResolver{Pkg: "check", Log: pkglog.WithContext(ctx).Logger} dialer := &net.Dialer{Timeout: 10 * time.Second} nctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -412,6 +412,8 @@ func (Admin) CheckDomain(ctx context.Context, domainName string) (r CheckResult) } func checkDomain(ctx context.Context, resolver dns.Resolver, dialer *net.Dialer, domainName string) (r CheckResult) { + log := pkglog.WithContext(ctx) + domain, err := dns.ParseDomain(domainName) xcheckuserf(ctx, err, "parsing domain") @@ -709,7 +711,7 @@ EOF cctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() err = conn.SetDeadline(end) - xlog.WithContext(ctx).Check(err, "setting deadline") + log.WithContext(ctx).Check(err, "setting deadline") br := bufio.NewReader(conn) _, err = br.ReadString('\n') @@ -901,7 +903,7 @@ EOF // Verify a domain with the configured IPs that do SMTP. verifySPF := func(kind string, domain dns.Domain) (string, *SPFRecord, spf.Record) { - _, txt, record, _, err := spf.Lookup(ctx, resolver, domain) + _, txt, record, _, err := spf.Lookup(ctx, log.Logger, resolver, domain) if err != nil { addf(&r.SPF.Errors, "Looking up %s SPF record: %s", kind, err) } @@ -933,7 +935,7 @@ EOF LocalIP: net.ParseIP("127.0.0.1"), LocalHostname: dns.Domain{ASCII: "localhost"}, } - status, mechanism, expl, _, err := spf.Evaluate(ctx, record, resolver, args) + status, mechanism, expl, _, err := spf.Evaluate(ctx, log.Logger, record, resolver, args) if err != nil { addf(&r.SPF.Errors, "Evaluating IP %q against %s SPF record: %s", ip, kind, err) } else if status != spf.StatusPass { @@ -998,7 +1000,7 @@ EOF haveEd25519 = true } - _, record, txt, _, err := dkim.Lookup(ctx, resolver, selc.Domain, domain) + _, record, txt, _, err := dkim.Lookup(ctx, log.Logger, resolver, selc.Domain, domain) if err != nil { missing = append(missing, sel) if errors.Is(err, dkim.ErrNoRecord) { @@ -1072,7 +1074,7 @@ EOF defer logPanic(ctx) defer wg.Done() - _, dmarcDomain, record, txt, _, err := dmarc.Lookup(ctx, resolver, domain) + _, dmarcDomain, record, txt, _, err := dmarc.Lookup(ctx, log.Logger, resolver, domain) if err != nil { addf(&r.DMARC.Errors, "Looking up DMARC record: %s", err) } else if record == nil { @@ -1102,10 +1104,10 @@ EOF // needs a special DNS record to opt-in to receiving reports. We check for that // record. // ../rfc/7489:1541 - orgDom := publicsuffix.Lookup(ctx, domain) - destOrgDom := publicsuffix.Lookup(ctx, domConf.DMARC.DNSDomain) + orgDom := publicsuffix.Lookup(ctx, log.Logger, domain) + destOrgDom := publicsuffix.Lookup(ctx, log.Logger, domConf.DMARC.DNSDomain) if orgDom != destOrgDom { - accepts, status, _, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, resolver, domain, domConf.DMARC.DNSDomain) + accepts, status, _, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, log.Logger, resolver, domain, domConf.DMARC.DNSDomain) if status != dmarc.StatusNone { addf(&r.DMARC.Errors, "Checking if external destination accepts reports: %s", err) } else if !accepts { @@ -1149,7 +1151,7 @@ EOF defer logPanic(ctx) defer wg.Done() - record, txt, err := tlsrpt.Lookup(ctx, resolver, dom) + record, txt, err := tlsrpt.Lookup(ctx, log.Logger, resolver, dom) if err != nil { addf(&result.Errors, "Looking up TLSRPT record: %s", err) } @@ -1225,7 +1227,7 @@ Ensure a DNS TXT record like the following exists: defer logPanic(ctx) defer wg.Done() - record, txt, err := mtasts.LookupRecord(ctx, resolver, domain) + record, txt, err := mtasts.LookupRecord(ctx, log.Logger, resolver, domain) if err != nil { addf(&r.MTASTS.Errors, "Looking up MTA-STS record: %s", err) } @@ -1234,7 +1236,7 @@ Ensure a DNS TXT record like the following exists: r.MTASTS.Record = &MTASTSRecord{*record} } - policy, text, err := mtasts.FetchPolicy(ctx, domain) + policy, text, err := mtasts.FetchPolicy(ctx, log.Logger, domain) if err != nil { addf(&r.MTASTS.Errors, "Fetching MTA-STS policy: %s", err) } else if policy.Mode == mtasts.ModeNone { @@ -1713,7 +1715,7 @@ type Reverse struct { // LookupIP does a reverse lookup of ip. func (Admin) LookupIP(ctx context.Context, ip string) Reverse { - resolver := dns.StrictResolver{Pkg: "webadmin"} + resolver := dns.StrictResolver{Pkg: "webadmin", Log: pkglog.WithContext(ctx).Logger} names, _, err := resolver.LookupAddr(ctx, ip) xcheckuserf(ctx, err, "looking up ip") return Reverse{names} @@ -1727,11 +1729,12 @@ func (Admin) LookupIP(ctx context.Context, ip string) Reverse { // The returned value maps IPs to per DNSBL statuses, where "pass" means not listed and // anything else is an error string, e.g. "fail: ..." or "temperror: ...". func (Admin) DNSBLStatus(ctx context.Context) map[string]map[string]string { - resolver := dns.StrictResolver{Pkg: "check"} - return dnsblsStatus(ctx, resolver) + log := mlog.New("webadmin", nil).WithContext(ctx) + resolver := dns.StrictResolver{Pkg: "check", Log: log.Logger} + return dnsblsStatus(ctx, log, resolver) } -func dnsblsStatus(ctx context.Context, resolver dns.Resolver) map[string]map[string]string { +func dnsblsStatus(ctx context.Context, log mlog.Log, resolver dns.Resolver) map[string]map[string]string { // todo: check health before using dnsbl? var dnsbls []dns.Domain if l, ok := mox.Conf.Static.Listeners["public"]; ok { @@ -1750,7 +1753,7 @@ func dnsblsStatus(ctx context.Context, resolver dns.Resolver) map[string]map[str ipstr := ip.String() r[ipstr] = map[string]string{} for _, zone := range dnsbls { - status, expl, err := dnsbl.Lookup(ctx, resolver, zone, ip) + status, expl, err := dnsbl.Lookup(ctx, log.Logger, resolver, zone, ip) result := string(status) if err != nil { result += ": " + err.Error() @@ -1773,7 +1776,7 @@ func (Admin) DomainRecords(ctx context.Context, domain string) []string { if !ok { xcheckuserf(ctx, errors.New("unknown domain"), "lookup domain") } - resolver := dns.StrictResolver{Pkg: "admin"} + resolver := dns.StrictResolver{Pkg: "webadmin", Log: pkglog.WithContext(ctx).Logger} _, result, err := resolver.LookupTXT(ctx, domain+".") if !dns.IsNotFound(err) { xcheckf(ctx, err, "looking up record to determine if dnssec is implemented") @@ -1830,16 +1833,17 @@ func (Admin) AddressRemove(ctx context.Context, address string) { // Sessions are not interrupted, and will keep working. New login attempts must use the new password. // Password must be at least 8 characters. func (Admin) SetPassword(ctx context.Context, accountName, password string) { + log := pkglog.WithContext(ctx) if len(password) < 8 { panic(&sherpa.Error{Code: "user:error", Message: "password must be at least 8 characters"}) } - acc, err := store.OpenAccount(accountName) + acc, err := store.OpenAccount(log, accountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() - xlog.Check(err, "closing account") + log.WithContext(ctx).Check(err, "closing account") }() - err = acc.SetPassword(password) + err = acc.SetPassword(log, password) xcheckf(ctx, err, "setting password") } @@ -1886,7 +1890,8 @@ func (Admin) QueueKick(ctx context.Context, id int64, transport string) { // QueueDrop removes a message from the queue. func (Admin) QueueDrop(ctx context.Context, id int64) { - n, err := queue.Drop(ctx, id, "", "") + log := pkglog.WithContext(ctx) + n, err := queue.Drop(ctx, log, id, "", "") if err == nil && n == 0 { err = errors.New("message not found") } @@ -1904,7 +1909,11 @@ func (Admin) QueueSaveRequireTLS(ctx context.Context, id int64, requireTLS *bool func (Admin) LogLevels(ctx context.Context) map[string]string { m := map[string]string{} for pkg, level := range mox.Conf.LogLevels() { - m[pkg] = level.String() + s, ok := mlog.LevelStrings[level] + if !ok { + s = level.String() + } + m[pkg] = s } return m } @@ -1915,12 +1924,12 @@ func (Admin) LogLevelSet(ctx context.Context, pkg string, levelStr string) { if !ok { xcheckuserf(ctx, errors.New("unknown"), "lookup level") } - mox.Conf.LogLevelSet(pkg, level) + mox.Conf.LogLevelSet(pkglog.WithContext(ctx), pkg, level) } // LogLevelRemove removes a log level for a package, which cannot be the empty string. func (Admin) LogLevelRemove(ctx context.Context, pkg string) { - mox.Conf.LogLevelRemove(pkg) + mox.Conf.LogLevelRemove(pkglog.WithContext(ctx), pkg) } // CheckUpdatesEnabled returns whether checking for updates is enabled. @@ -2086,11 +2095,12 @@ func (Admin) TLSRPTResultsDomain(ctx context.Context, isRcptDom bool, policyDoma // LookupTLSRPTRecord looks up a TLSRPT record and returns the parsed form, original txt // form from DNS, and error with the TLSRPT record as a string. func (Admin) LookupTLSRPTRecord(ctx context.Context, domain string) (record *TLSRPTRecord, txt string, errstr string) { + log := pkglog.WithContext(ctx) dom, err := dns.ParseDomain(domain) xcheckf(ctx, err, "parsing domain") - resolver := dns.StrictResolver{Pkg: "webadmin"} - r, txt, err := tlsrpt.Lookup(ctx, resolver, dom) + resolver := dns.StrictResolver{Pkg: "webadmin", Log: log.Logger} + r, txt, err := tlsrpt.Lookup(ctx, log.Logger, resolver, dom) if err != nil && (errors.Is(err, tlsrpt.ErrNoRecord) || errors.Is(err, tlsrpt.ErrMultipleRecords) || errors.Is(err, tlsrpt.ErrRecordSyntax) || errors.Is(err, tlsrpt.ErrDNS)) { errstr = err.Error() err = nil diff --git a/webadmin/admin_test.go b/webadmin/admin_test.go index 07e1997..4f3bc41 100644 --- a/webadmin/admin_test.go +++ b/webadmin/admin_test.go @@ -13,6 +13,7 @@ import ( "github.com/mjl-/mox/config" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" ) @@ -67,6 +68,8 @@ func TestAdminAuth(t *testing.T) { func TestCheckDomain(t *testing.T) { // NOTE: we aren't currently looking at the results, having the code paths executed is better than nothing. + log := mlog.New("webadmin", nil) + resolver := dns.MockResolver{ MX: map[string][]*net.MX{ "mox.example.": {{Host: "mail.mox.example.", Pref: 10}}, @@ -130,6 +133,6 @@ func TestCheckDomain(t *testing.T) { checkDomain(ctxbg, resolver, dialer, "mox.example") // todo: check returned data - Admin{}.Domains(ctxbg) // todo: check results - dnsblsStatus(ctxbg, resolver) // todo: check results + Admin{}.Domains(ctxbg) // todo: check results + dnsblsStatus(ctxbg, log, resolver) // todo: check results } diff --git a/webmail/api.go b/webmail/api.go index 3850cf7..fe81eed 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -23,6 +23,7 @@ import ( _ "embed" "golang.org/x/exp/maps" + "golang.org/x/exp/slog" "github.com/mjl-/bstore" "github.com/mjl-/sherpa" @@ -33,7 +34,6 @@ import ( "github.com/mjl-/mox/dns" "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" - "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxvar" @@ -55,7 +55,7 @@ type Webmail struct { func mustParseAPI(api string, buf []byte) (doc sherpadoc.Section) { err := json.Unmarshal(buf, &doc) if err != nil { - xlog.Fatalx("parsing webmail api docs", err, mlog.Field("api", api)) + pkglog.Fatalx("parsing webmail api docs", err, slog.String("api", api)) } return doc } @@ -71,14 +71,14 @@ func makeSherpaHandler(maxMessageSize int64) (http.Handler, error) { func init() { collector, err := sherpaprom.NewCollector("moxwebmail", nil) if err != nil { - xlog.Fatalx("creating sherpa prometheus collector", err) + pkglog.Fatalx("creating sherpa prometheus collector", err) } sherpaHandlerOpts = &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none"} // Just to validate. _, err = makeSherpaHandler(0) if err != nil { - xlog.Fatalx("sherpa handler", err) + pkglog.Fatalx("sherpa handler", err) } } @@ -112,9 +112,9 @@ func (Webmail) Request(ctx context.Context, req Request) { // ParsedMessage returns enough to render the textual body of a message. It is // assumed the client already has other fields through MessageItem. func (Webmail) ParsedMessage(ctx context.Context, msgID int64) (pm ParsedMessage) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -234,8 +234,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { // todo: consider making this an HTTP POST, so we can upload as regular form, which is probably more efficient for encoding for the client and we can stream the data in. reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - log := xlog.WithContext(ctx).Fields(mlog.Field("account", reqInfo.AccountName)) - acc, err := store.OpenAccount(reqInfo.AccountName) + log := pkglog.WithContext(ctx).With(slog.String("account", reqInfo.AccountName)) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -326,7 +326,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { } // Create file to compose message into. - dataFile, err := store.CreateMessageTemp("webmail-submit") + dataFile, err := store.CreateMessageTemp(log, "webmail-submit") xcheckf(ctx, err, "creating temporary file for message") defer store.CloseRemoveTempFile(log, dataFile, "message to submit") @@ -361,7 +361,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { // Note: we don't have "via" or "with", there is no registered for webmail. recvHdr.Add(" ", "Received:", "from", recvFrom, "by", recvBy, "id", recvID) // ../rfc/5321:3158 if reqInfo.Request.TLS != nil { - recvHdr.Add(" ", message.TLSReceivedComment(log, *reqInfo.Request.TLS)...) + recvHdr.Add(" ", mox.TLSReceivedComment(log, *reqInfo.Request.TLS)...) } recvHdr.Add(" ", "for", "<"+rcptTo+">;", time.Now().Format(message.RFC5322Z)) return recvHdr.String() @@ -558,7 +558,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { fd := fromAddr.Address.Domain confDom, _ := mox.Conf.Domain(fd) if len(confDom.DKIM.Sign) > 0 { - dkimHeaders, err := dkim.Sign(ctx, fromAddr.Address.Localpart, fd, confDom.DKIM, smtputf8, dataFile) + dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Address.Localpart, fd, confDom.DKIM, smtputf8, dataFile) if err != nil { metricServerErrors.WithLabelValues("dkimsign").Inc() } @@ -671,9 +671,9 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { // MessageMove moves messages to another mailbox. If the message is already in // the mailbox an error is returned. func (Webmail) MessageMove(ctx context.Context, messageIDs []int64, mailboxID int64) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -802,9 +802,9 @@ func (Webmail) MessageMove(ctx context.Context, messageIDs []int64, mailboxID in // MessageDelete permanently deletes messages, without moving them to the Trash mailbox. func (Webmail) MessageDelete(ctx context.Context, messageIDs []int64) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -895,9 +895,9 @@ func (Webmail) MessageDelete(ctx context.Context, messageIDs []int64) { // FlagsAdd adds flags, either system flags like \Seen or custom keywords. The // flags should be lower-case, but will be converted and verified. func (Webmail) FlagsAdd(ctx context.Context, messageIDs []int64, flaglist []string) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -978,9 +978,9 @@ func (Webmail) FlagsAdd(ctx context.Context, messageIDs []int64, flaglist []stri // FlagsClear clears flags, either system flags like \Seen or custom keywords. func (Webmail) FlagsClear(ctx context.Context, messageIDs []int64, flaglist []string) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1056,9 +1056,9 @@ func (Webmail) FlagsClear(ctx context.Context, messageIDs []int64, flaglist []st // MailboxCreate creates a new mailbox. func (Webmail) MailboxCreate(ctx context.Context, name string) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1086,9 +1086,9 @@ func (Webmail) MailboxCreate(ctx context.Context, name string) { // MailboxDelete deletes a mailbox and all its messages. func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1123,16 +1123,16 @@ func (Webmail) MailboxDelete(ctx context.Context, mailboxID int64) { for _, mID := range removeMessageIDs { p := acc.MessagePath(mID) err := os.Remove(p) - log.Check(err, "removing message file for mailbox delete", mlog.Field("path", p)) + log.Check(err, "removing message file for mailbox delete", slog.String("path", p)) } } // MailboxEmpty empties a mailbox, removing all messages from the mailbox, but not // its child mailboxes. func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1196,16 +1196,16 @@ func (Webmail) MailboxEmpty(ctx context.Context, mailboxID int64) { for _, m := range expunged { p := acc.MessagePath(m.ID) err := os.Remove(p) - log.Check(err, "removing message file after emptying mailbox", mlog.Field("path", p)) + log.Check(err, "removing message file after emptying mailbox", slog.String("path", p)) } } // MailboxRename renames a mailbox, possibly moving it to a new parent. The mailbox // ID and its messages are unchanged. func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName string) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1239,9 +1239,9 @@ func (Webmail) MailboxRename(ctx context.Context, mailboxID int64, newName strin // matches, most recently used first, and whether this is the full list and further // requests for longer prefixes aren't necessary. func (Webmail) CompleteRecipient(ctx context.Context, search string) ([]string, bool) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1342,9 +1342,9 @@ func addressString(a message.Address, smtputf8 bool) string { // MailboxSetSpecialUse sets the special use flags of a mailbox. func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1395,9 +1395,9 @@ func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) { // children. The messageIDs are typically thread roots. But not all roots // (without parent) of a thread need to have the same collapsed state. func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse bool) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1455,9 +1455,9 @@ func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse // ThreadMute saves the ThreadMute field for the messages and their children. // If messages are muted, they are also marked collapsed. func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1555,7 +1555,7 @@ type RecipientSecurity struct { // RecipientSecurity looks up security properties of the address in the // single-address message addressee (as it appears in a To/Cc/Bcc/etc header). func (Webmail) RecipientSecurity(ctx context.Context, messageAddressee string) (RecipientSecurity, error) { - resolver := dns.StrictResolver{Pkg: "webmail"} + resolver := dns.StrictResolver{Pkg: "webmail", Log: pkglog.WithContext(ctx).Logger} return recipientSecurity(ctx, resolver, messageAddressee) } @@ -1565,15 +1565,15 @@ func logPanic(ctx context.Context) { if x == nil { return } - log := xlog.WithContext(ctx) - log.Error("recover from panic", mlog.Field("panic", x)) + log := pkglog.WithContext(ctx) + log.Error("recover from panic", slog.Any("panic", x)) debug.PrintStack() metrics.PanicInc(metrics.Webmail) } // separate function for testing with mocked resolver. func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddressee string) (RecipientSecurity, error) { - log := xlog.WithContext(ctx) + log := pkglog.WithContext(ctx) rs := RecipientSecurity{ SecurityResultUnknown, @@ -1601,7 +1601,7 @@ func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddres defer logPanic(ctx) defer wg.Done() - policy, _, _, err := mtastsdb.Get(ctx, resolver, addr.Domain) + policy, _, _, err := mtastsdb.Get(ctx, log.Logger, resolver, addr.Domain) if policy != nil && policy.Mode == mtasts.ModeEnforce { rs.MTASTS = SecurityResultYes } else if err == nil { @@ -1617,7 +1617,7 @@ func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddres defer logPanic(ctx) defer wg.Done() - _, origNextHopAuthentic, expandedNextHopAuthentic, _, hosts, _, err := smtpclient.GatherDestinations(ctx, log, resolver, dns.IPDomain{Domain: addr.Domain}) + _, origNextHopAuthentic, expandedNextHopAuthentic, _, hosts, _, err := smtpclient.GatherDestinations(ctx, log.Logger, resolver, dns.IPDomain{Domain: addr.Domain}) if err != nil { rs.DNSSEC = SecurityResultError return @@ -1641,7 +1641,7 @@ func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddres // Resolve the IPs. Required for DANE to prevent bad DNS servers from causing an // error result instead of no-DANE result. - authentic, expandedAuthentic, expandedHost, _, _, err := smtpclient.GatherIPs(ctx, log, resolver, host, map[string][]net.IP{}) + authentic, expandedAuthentic, expandedHost, _, _, err := smtpclient.GatherIPs(ctx, log.Logger, resolver, host, map[string][]net.IP{}) if err != nil { rs.DANE = SecurityResultError return @@ -1651,7 +1651,7 @@ func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddres return } - daneRequired, _, _, err := smtpclient.GatherTLSA(ctx, log, resolver, host.Domain, expandedAuthentic, expandedHost) + daneRequired, _, _, err := smtpclient.GatherTLSA(ctx, log.Logger, resolver, host.Domain, expandedAuthentic, expandedHost) if err != nil { rs.DANE = SecurityResultError return @@ -1664,7 +1664,7 @@ func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddres // STARTTLS and RequireTLS reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) - acc, err := store.OpenAccount(reqInfo.AccountName) + acc, err := store.OpenAccount(log, reqInfo.AccountName) xcheckf(ctx, err, "open account") defer func() { if acc != nil { @@ -1682,7 +1682,7 @@ func recipientSecurity(ctx context.Context, resolver dns.Resolver, messageAddres } else if err != nil { rs.STARTTLS = SecurityResultError rs.RequireTLS = SecurityResultError - log.Errorx("looking up recipient domain", err, mlog.Field("domain", addr.Domain)) + log.Errorx("looking up recipient domain", err, slog.Any("domain", addr.Domain)) return nil } if rd.STARTTLS { diff --git a/webmail/api_test.go b/webmail/api_test.go index 915ff8e..c312565 100644 --- a/webmail/api_test.go +++ b/webmail/api_test.go @@ -13,6 +13,7 @@ import ( "github.com/mjl-/sherpa" "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/queue" "github.com/mjl-/mox/store" @@ -49,13 +50,14 @@ func TestAPI(t *testing.T) { mox.MustLoadConfig(true, false) defer store.Switchboard()() - acc, err := store.OpenAccount("mjl") + log := mlog.New("webmail", nil) + acc, err := store.OpenAccount(log, "mjl") tcheck(t, err, "open account") - err = acc.SetPassword("test1234") + err = acc.SetPassword(log, "test1234") tcheck(t, err, "set password") defer func() { err := acc.Close() - xlog.Check(err, "closing account") + pkglog.Check(err, "closing account") }() var zerom store.Message diff --git a/webmail/eventwriter.go b/webmail/eventwriter.go index 715bdcf..5f6459a 100644 --- a/webmail/eventwriter.go +++ b/webmail/eventwriter.go @@ -12,6 +12,8 @@ import ( "sync" "time" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/metrics" "github.com/mjl-/mox/mlog" ) @@ -69,7 +71,7 @@ var waitGen = mathrand.New(mathrand.NewSource(time.Now().UnixNano())) // Schedule an event for writing to the connection. If events get a delay, this // function still returns immediately. -func (ew *eventWriter) xsendEvent(ctx context.Context, log *mlog.Log, name string, v any) { +func (ew *eventWriter) xsendEvent(ctx context.Context, log mlog.Log, name string, v any) { if (ew.waitMin > 0 || ew.waitMax > 0) && ew.events == nil { // First write on a connection with delay. ew.events = make(chan struct { @@ -82,7 +84,7 @@ func (ew *eventWriter) xsendEvent(ctx context.Context, log *mlog.Log, name strin defer func() { x := recover() // Should not happen, but don't take program down if it does. if x != nil { - log.WithContext(ctx).Error("writeEvent panic", mlog.Field("err", x)) + log.WithContext(ctx).Error("writeEvent panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Webmailsendevent) } diff --git a/webmail/message.go b/webmail/message.go index 139a4d9..76f7a3b 100644 --- a/webmail/message.go +++ b/webmail/message.go @@ -8,6 +8,8 @@ import ( "net/url" "strings" + "golang.org/x/exp/slog" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/message" "github.com/mjl-/mox/mlog" @@ -32,19 +34,19 @@ import ( // "filename*0*=UTF-8”%...; filename*1*=%.... We'll look for Q/B-word encoding // markers ("=?"-prefix or "?="-suffix) and try to decode if present. This would // only cause trouble for filenames having this prefix/suffix. -func tryDecodeParam(log *mlog.Log, name string) string { +func tryDecodeParam(log mlog.Log, name string) string { if name == "" || !strings.HasPrefix(name, "=?") && !strings.HasSuffix(name, "?=") { return name } // todo: find where this is allowed. it seems quite common. perhaps we should remove the pedantic check? if moxvar.Pedantic { - log.Debug("attachment contains rfc2047 q/b-word-encoded mime parameter instead of rfc2231-encoded", mlog.Field("name", name)) + log.Debug("attachment contains rfc2047 q/b-word-encoded mime parameter instead of rfc2231-encoded", slog.String("name", name)) return name } dec := mime.WordDecoder{} s, err := dec.DecodeHeader(name) if err != nil { - log.Debugx("q/b-word decoding mime parameter", err, mlog.Field("name", name)) + log.Debugx("q/b-word decoding mime parameter", err, slog.String("name", name)) return name } return s @@ -52,7 +54,7 @@ func tryDecodeParam(log *mlog.Log, name string) string { // todo: mime.FormatMediaType does not wrap long lines. should do it ourselves, and split header into several parts (if commonly supported). -func messageItem(log *mlog.Log, m store.Message, state *msgState) (MessageItem, error) { +func messageItem(log mlog.Log, m store.Message, state *msgState) (MessageItem, error) { pm, err := parsedMessage(log, m, state, false, true) if err != nil { return MessageItem{}, fmt.Errorf("parsing message %d for item: %v", m.ID, err) @@ -161,7 +163,7 @@ func formatFirstLine(r io.Reader) (string, error) { return result, scanner.Err() } -func parsedMessage(log *mlog.Log, m store.Message, state *msgState, full, msgitem bool) (pm ParsedMessage, rerr error) { +func parsedMessage(log mlog.Log, m store.Message, state *msgState, full, msgitem bool) (pm ParsedMessage, rerr error) { if full || msgitem { if !state.ensurePart(m, true) { return pm, state.err @@ -240,11 +242,11 @@ func parsedMessage(log *mlog.Log, m store.Message, state *msgState, full, msgite if full || msgitem { // todo: should have this, and perhaps all content-* headers, preparsed in message.Part? h, err := p.Header() - log.Check(err, "parsing attachment headers", mlog.Field("msgid", m.ID)) + log.Check(err, "parsing attachment headers", slog.Int64("msgid", m.ID)) cp := h.Get("Content-Disposition") if cp != "" { disp, params, err := mime.ParseMediaType(cp) - log.Check(err, "parsing content-disposition", mlog.Field("cp", cp)) + log.Check(err, "parsing content-disposition", slog.String("cp", cp)) if strings.EqualFold(disp, "attachment") { name := tryDecodeParam(log, p.ContentTypeParams["name"]) if name == "" { @@ -322,11 +324,11 @@ func parsedMessage(log *mlog.Log, m store.Message, state *msgState, full, msgite if name == "" && (full || msgitem) { // todo: should have this, and perhaps all content-* headers, preparsed in message.Part? h, err := p.Header() - log.Check(err, "parsing attachment headers", mlog.Field("msgid", m.ID)) + log.Check(err, "parsing attachment headers", slog.Int64("msgid", m.ID)) cp := h.Get("Content-Disposition") if cp != "" { _, params, err := mime.ParseMediaType(cp) - log.Check(err, "parsing content-disposition", mlog.Field("cp", cp)) + log.Check(err, "parsing content-disposition", slog.String("cp", cp)) name = tryDecodeParam(log, params["filename"]) } } diff --git a/webmail/view.go b/webmail/view.go index e2770cb..fc00bf1 100644 --- a/webmail/view.go +++ b/webmail/view.go @@ -20,6 +20,7 @@ import ( "time" "golang.org/x/exp/slices" + "golang.org/x/exp/slog" "github.com/mjl-/bstore" "github.com/mjl-/sherpa" @@ -486,7 +487,7 @@ type ioErr struct { // serveEvents serves an SSE connection. Authentication is done through a query // string parameter "token", a one-time-use token returned by the Token API call. -func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *http.Request) { +func serveEvents(ctx context.Context, log mlog.Log, w http.ResponseWriter, r *http.Request) { if r.Method != "GET" { http.Error(w, "405 - method not allowed - use get", http.StatusMethodNotAllowed) return @@ -569,7 +570,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h } else if _, ok := x.(ioErr); ok { return } else { - log.WithContext(ctx).Error("serveEvents panic", mlog.Field("err", x)) + log.WithContext(ctx).Error("serveEvents panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Webmail) panic(x) @@ -597,7 +598,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h defer writer.close() // Fetch initial data. - acc, err := store.OpenAccount(accName) + acc, err := store.OpenAccount(log, accName) xcheckf(ctx, err, "open account") defer func() { err := acc.Close() @@ -1123,7 +1124,7 @@ func (v view) inRange(m store.Message) bool { // message). getmsg retrieves the message, which may be necessary depending on the // active filters. Used to determine if a store.Change with a new message should be // sent, and for the destination and anchor messages in view requests. -func (v view) matches(log *mlog.Log, acc *store.Account, checkRange bool, messageID int64, mailboxID int64, uid store.UID, flags store.Flags, keywords []string, getmsg func(int64, int64, store.UID) (store.Message, error)) (match bool, rerr error) { +func (v view) matches(log mlog.Log, acc *store.Account, checkRange bool, messageID int64, mailboxID int64, uid store.UID, flags store.Flags, keywords []string, getmsg func(int64, int64, store.UID) (store.Message, error)) (match bool, rerr error) { var m store.Message ensureMessage := func() bool { if m.ID == 0 && rerr == nil { @@ -1208,7 +1209,7 @@ type msgResp struct { // and sending Event* to the SSE connection. // // It always closes tx. -func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *bstore.Tx, v view, msgc chan EventViewMsgs, errc chan EventViewErr, resetc chan EventViewReset, donec chan int64) { +func viewRequestTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, v view, msgc chan EventViewMsgs, errc chan EventViewErr, resetc chan EventViewReset, donec chan int64) { defer func() { err := tx.Rollback() log.Check(err, "rolling back query transaction") @@ -1217,7 +1218,7 @@ func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b x := recover() // Should not happen, but don't take program down if it does. if x != nil { - log.WithContext(ctx).Error("viewRequestTx panic", mlog.Field("err", x)) + log.WithContext(ctx).Error("viewRequestTx panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Webmailrequest) } @@ -1296,11 +1297,11 @@ func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b // It sends on msgc, with several types of messages: errors, whether the view is // reset due to missing AnchorMessageID, and when the end of the view was reached // and/or for a message. -func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *bstore.Tx, v view, mrc chan msgResp) { +func queryMessages(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, v view, mrc chan msgResp) { defer func() { x := recover() // Should not happen, but don't take program down if it does. if x != nil { - log.WithContext(ctx).Error("queryMessages panic", mlog.Field("err", x)) + log.WithContext(ctx).Error("queryMessages panic", slog.Any("err", x)) debug.PrintStack() mrc <- msgResp{err: fmt.Errorf("query failed")} metrics.PanicInc(metrics.Webmailquery) @@ -1542,7 +1543,7 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b } } -func gatherThread(log *mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m store.Message, destMessageID int64, first bool) ([]MessageItem, *ParsedMessage, error) { +func gatherThread(log mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m store.Message, destMessageID int64, first bool) ([]MessageItem, *ParsedMessage, error) { if m.ThreadID == 0 { // If we would continue, FilterNonzero would fail because there are no non-zero fields. return nil, nil, fmt.Errorf("message has threadid 0, account is probably still being upgraded, try turning threading off until the upgrade is done") @@ -1717,7 +1718,7 @@ func (q Query) flagFilterFn() func(store.Flags, []string) bool { // attachmentFilterFn returns a function that filters for the attachment-related // filter from the query. A nil function is returned if there are attachment // filters. -func (q Query) attachmentFilterFn(log *mlog.Log, acc *store.Account, state *msgState) func(m store.Message) bool { +func (q Query) attachmentFilterFn(log mlog.Log, acc *store.Account, state *msgState) func(m store.Message) bool { if q.Filter.Attachments == AttachmentIndifferent && q.NotFilter.Attachments == AttachmentIndifferent { return nil } @@ -1774,7 +1775,7 @@ var attachmentExtensions = map[string]AttachmentType{ ".pptx": AttachmentPresentation, } -func attachmentTypes(log *mlog.Log, m store.Message, state *msgState) (map[AttachmentType]bool, error) { +func attachmentTypes(log mlog.Log, m store.Message, state *msgState) (map[AttachmentType]bool, error) { types := map[AttachmentType]bool{} pm, err := parsedMessage(log, m, state, false, false) @@ -1810,7 +1811,7 @@ func attachmentTypes(log *mlog.Log, m store.Message, state *msgState) (map[Attac // used by IMAP, i.e. basic message headers from/to/subject, an unfortunate name // clash with SMTP envelope) for the query. A nil function is returned if no // filtering is needed. -func (q Query) envFilterFn(log *mlog.Log, state *msgState) func(m store.Message) bool { +func (q Query) envFilterFn(log mlog.Log, state *msgState) func(m store.Message) bool { if len(q.Filter.From) == 0 && len(q.Filter.To) == 0 && len(q.Filter.Subject) == 0 && len(q.NotFilter.From) == 0 && len(q.NotFilter.To) == 0 && len(q.NotFilter.Subject) == 0 { return nil } @@ -1898,7 +1899,7 @@ func (q Query) envFilterFn(log *mlog.Log, state *msgState) func(m store.Message) // headerFilterFn returns a function that filters for the header filters in the // query. A nil function is returned if there are no header filters. -func (q Query) headerFilterFn(log *mlog.Log, state *msgState) func(m store.Message) bool { +func (q Query) headerFilterFn(log mlog.Log, state *msgState) func(m store.Message) bool { if len(q.Filter.Headers) == 0 { return nil } @@ -1939,7 +1940,7 @@ func (q Query) headerFilterFn(log *mlog.Log, state *msgState) func(m store.Messa // wordFiltersFn returns a function that applies the word filters of the query. A // nil function is returned when query does not contain a word filter. -func (q Query) wordsFilterFn(log *mlog.Log, state *msgState) func(m store.Message) bool { +func (q Query) wordsFilterFn(log mlog.Log, state *msgState) func(m store.Message) bool { if len(q.Filter.Words) == 0 && len(q.NotFilter.Words) == 0 { return nil } diff --git a/webmail/view_test.go b/webmail/view_test.go index 25f33bb..f40a8ba 100644 --- a/webmail/view_test.go +++ b/webmail/view_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) @@ -28,13 +29,14 @@ func TestView(t *testing.T) { mox.MustLoadConfig(true, false) defer store.Switchboard()() - acc, err := store.OpenAccount("mjl") + log := mlog.New("webmail", nil) + acc, err := store.OpenAccount(log, "mjl") tcheck(t, err, "open account") - err = acc.SetPassword("test1234") + err = acc.SetPassword(log, "test1234") tcheck(t, err, "set password") defer func() { err := acc.Close() - xlog.Check(err, "closing account") + pkglog.Check(err, "closing account") }() api := Webmail{maxMessageSize: 1024 * 1024} @@ -465,7 +467,7 @@ type eventReader struct { func (r eventReader) Get(name string, event any) { timer := time.AfterFunc(2*time.Second, func() { r.r.Close() - xlog.Print("event timeout") + pkglog.Print("event timeout") }) defer timer.Stop() diff --git a/webmail/webmail.go b/webmail/webmail.go index c00b4ef..b331444 100644 --- a/webmail/webmail.go +++ b/webmail/webmail.go @@ -26,11 +26,12 @@ import ( _ "embed" + "golang.org/x/exp/slog" + "golang.org/x/net/html" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "golang.org/x/net/html" - "github.com/mjl-/bstore" "github.com/mjl-/sherpa" @@ -48,7 +49,7 @@ func init() { mox.LimitersInit() } -var xlog = mlog.New("webmail") +var pkglog = mlog.New("webmail", nil) // We pass the request to the sherpa handler so the TLS info can be used for // the Received header in submitted messages. Most API calls need just the @@ -115,7 +116,7 @@ func xcheckf(ctx context.Context, err error, format string, args ...any) { } msg := fmt.Sprintf(format, args...) errmsg := fmt.Sprintf("%s: %s", msg, err) - xlog.WithContext(ctx).Errorx(msg, err) + pkglog.WithContext(ctx).Errorx(msg, err) panic(&sherpa.Error{Code: "server:error", Message: errmsg}) } @@ -125,7 +126,7 @@ func xcheckuserf(ctx context.Context, err error, format string, args ...any) { } msg := fmt.Sprintf(format, args...) errmsg := fmt.Sprintf("%s: %s", msg, err) - xlog.WithContext(ctx).Errorx(msg, err) + pkglog.WithContext(ctx).Errorx(msg, err) panic(&sherpa.Error{Code: "user:error", Message: errmsg}) } @@ -166,7 +167,7 @@ var webmail = &merged{ // fallbackMtime returns a time to use for the Last-Modified header in case we // cannot find a file, e.g. when used in production. -func fallbackMtime(log *mlog.Log) time.Time { +func fallbackMtime(log mlog.Log) time.Time { p, err := os.Executable() log.Check(err, "finding executable for mtime") if err == nil { @@ -180,7 +181,7 @@ func fallbackMtime(log *mlog.Log) time.Time { return time.Now() } -func (m *merged) serve(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *http.Request) { +func (m *merged) serve(ctx context.Context, log mlog.Log, w http.ResponseWriter, r *http.Request) { // We typically return the embedded file, but during development it's handy // to load from disk. fhtml, _ := os.Open(m.htmlPath) @@ -304,7 +305,7 @@ func (w gzipInjector) WriteHeader(statusCode int) { // should already have set the content-type. We use this to return a file from // the local file system (during development), or embedded in the binary (when // deployed). -func serveContentFallback(log *mlog.Log, w http.ResponseWriter, r *http.Request, path string, fallback []byte) { +func serveContentFallback(log mlog.Log, w http.ResponseWriter, r *http.Request, path string, fallback []byte) { f, err := os.Open(path) if err == nil { defer f.Close() @@ -332,7 +333,7 @@ func Handler(maxMessageSize int64) func(w http.ResponseWriter, r *http.Request) func handle(apiHandler http.Handler, w http.ResponseWriter, r *http.Request) { ctx := r.Context() - log := xlog.WithContext(ctx).Fields(mlog.Field("userauth", "")) + log := pkglog.WithContext(ctx).With(slog.String("userauth", "")) // Server-sent event connection, for all initial data (list of mailboxes), list of // messages, and all events afterwards. Authenticated through a token in the query @@ -349,8 +350,8 @@ func handle(apiHandler http.Handler, w http.ResponseWriter, r *http.Request) { return } - if lw, ok := w.(interface{ AddField(f mlog.Pair) }); ok { - lw.AddField(mlog.Field("authaccount", accName)) + if lw, ok := w.(interface{ AddAttr(a slog.Attr) }); ok { + lw.AddAttr(slog.String("authaccount", accName)) } defer func() { @@ -360,7 +361,7 @@ func handle(apiHandler http.Handler, w http.ResponseWriter, r *http.Request) { } err, ok := x.(*sherpa.Error) if !ok { - log.WithContext(ctx).Error("handle panic", mlog.Field("err", x)) + log.WithContext(ctx).Error("handle panic", slog.Any("err", x)) debug.PrintStack() metrics.PanicInc(metrics.Webmailhandle) panic(x) @@ -462,7 +463,7 @@ func handle(apiHandler http.Handler, w http.ResponseWriter, r *http.Request) { var err error - acc, err = store.OpenAccount(accName) + acc, err = store.OpenAccount(log, accName) xcheckf(ctx, err, "open account") m = store.Message{ID: id} @@ -922,7 +923,7 @@ func acceptsGzip(r *http.Request) bool { // HTML, setHeaders is called to write the required headers for content-type and // CSP. On error, setHeader is not called, no output is written and the caller // should write an error response. -func inlineSanitizeHTML(log *mlog.Log, setHeaders func(), w io.Writer, p *message.Part, parents []*message.Part) error { +func inlineSanitizeHTML(log mlog.Log, setHeaders func(), w io.Writer, p *message.Part, parents []*message.Part) error { // Prepare cids if there is a chance we will use them. cids := map[string]*message.Part{} for _, parent := range parents { diff --git a/webmail/webmail_test.go b/webmail/webmail_test.go index c7d5eb1..e6275fd 100644 --- a/webmail/webmail_test.go +++ b/webmail/webmail_test.go @@ -252,14 +252,14 @@ type testmsg struct { } func tdeliver(t *testing.T, acc *store.Account, tm *testmsg) { - msgFile, err := store.CreateMessageTemp("webmail-test") + msgFile, err := store.CreateMessageTemp(pkglog, "webmail-test") tcheck(t, err, "create message temp") defer os.Remove(msgFile.Name()) defer msgFile.Close() size, err := msgFile.Write(tm.msg.Marshal(t)) tcheck(t, err, "write message temp") m := store.Message{Flags: tm.Flags, Keywords: tm.Keywords, Size: int64(size)} - err = acc.DeliverMailbox(xlog, tm.Mailbox, &m, msgFile) + err = acc.DeliverMailbox(pkglog, tm.Mailbox, &m, msgFile) tcheck(t, err, "deliver test message") err = msgFile.Close() tcheck(t, err, "closing test message") @@ -279,13 +279,13 @@ func TestWebmail(t *testing.T) { mox.MustLoadConfig(true, false) defer store.Switchboard()() - acc, err := store.OpenAccount("mjl") + acc, err := store.OpenAccount(pkglog, "mjl") tcheck(t, err, "open account") - err = acc.SetPassword("test1234") + err = acc.SetPassword(pkglog, "test1234") tcheck(t, err, "set password") defer func() { err := acc.Close() - xlog.Check(err, "closing account") + pkglog.Check(err, "closing account") }() api := Webmail{maxMessageSize: 1024 * 1024}