diff --git a/README.md b/README.md index 50eb212..1d25e97 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ See Quickstart below to get started. - Webserver with serving static files and forwarding requests (reverse proxy), so port 443 can also be used to serve websites. - Prometheus metrics and structured logging for operational insight. +- "localserve" subcommand for running mox locally for email-related + testing/developing. Mox is available under the MIT-license and was created by Mechiel Lukkien, mechiel@ueber.net. Mox includes the Public Suffix List by Mozilla, under Mozilla @@ -109,8 +111,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili - Strict vs lax mode, defaulting to lax when receiving from the internet, and strict when sending. -- "developer server" mode, to easily launch a local SMTP/IMAP server to test - your apps mail sending capabilities. - Rate limiting and spam detection for submitted/outgoing messages, to reduce impact when an account gets compromised. - Privilege separation, isolating parts of the application to more restricted @@ -118,7 +118,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili - DANE and DNSSEC. - Sending DMARC and TLS reports (currently only receiving). - OAUTH2 support, for single sign on. -- ACME verification over HTTP (in addition to current tls-alpn01). - Add special IMAP mailbox ("Queue?") that contains queued but not-yet-delivered messages. - Sieve for filtering (for now see Rulesets in the account config) diff --git a/doc.go b/doc.go index 27e996f..87d52ec 100644 --- a/doc.go +++ b/doc.go @@ -27,6 +27,7 @@ low-maintenance self-hosted email. mox import mbox accountname mailboxname mbox mox export maildir dst-dir account-path [mailbox] mox export mbox dst-dir account-path [mailbox] + mox localserve mox help [command ...] mox config test mox config dnscheck domain @@ -291,6 +292,36 @@ otherwise reconstructing the original could lose a ">". usage: mox export mbox dst-dir account-path [mailbox] +# mox localserve + +Start a local SMTP/IMAP server that accepts all messages, useful when testing/developing software that sends email. + +Localserve starts mox with a configuration suitable for local email-related +software development/testing. It listens for SMTP/Submission(s), IMAP(s) and +HTTP(s), on the regular port numbers + 1000. + +Data is stored in the system user's configuration directory under +"mox-localserve", e.g. $HOME/.config/mox-localserve/ on linux, but can be +overridden with the -dir flag. If the directory does not yet exist, it is +automatically initialized with configuration files, an account with email +address mox@localhost and password moxmoxmox, and a newly generated self-signed +TLS certificate. + +All incoming email is accepted (if checks pass), unless the recipient localpart +ends with: + +- "temperror": fail with a temporary error code +- "permerror": fail with a permanent error code +- [45][0-9][0-9]: fail with the specific error code +- "timeout": no response (for an hour) + +If the localpart begins with "mailfrom" or "rcptto", the error is returned +during those commands instead of during "data". + + usage: mox localserve + -dir string + configuration storage directory (default "$userconfigdir/mox-localserve") + # mox help Prints help about matching commands. diff --git a/gendoc.sh b/gendoc.sh index 4647c7e..f1ab779 100755 --- a/gendoc.sh +++ b/gendoc.sh @@ -28,7 +28,9 @@ directory) through the -config flag or MOXCONF environment variable. EOF -./mox helpall 2>&1 +# setting XDG_CONFIG_HOME ensures "mox localserve" has reasonable default +# values in its help output. +XDG_CONFIG_HOME='$userconfigdir' ./mox helpall 2>&1 cat < 0 { + if errs := mox.LoadConfig(context.Background(), false); len(errs) > 0 { t.Fatalf("loading mox config: %v", errs) } diff --git a/localserve.go b/localserve.go new file mode 100644 index 0000000..6e21d23 --- /dev/null +++ b/localserve.go @@ -0,0 +1,405 @@ +package main + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + cryptorand "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + golog "log" + "math/big" + "net" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "golang.org/x/crypto/bcrypt" + + "github.com/mjl-/sconf" + + "github.com/mjl-/mox/config" + "github.com/mjl-/mox/junk" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/queue" + "github.com/mjl-/mox/smtpserver" + "github.com/mjl-/mox/store" +) + +func cmdLocalserve(c *cmd) { + c.help = `Start a local SMTP/IMAP server that accepts all messages, useful when testing/developing software that sends email. + +Localserve starts mox with a configuration suitable for local email-related +software development/testing. It listens for SMTP/Submission(s), IMAP(s) and +HTTP(s), on the regular port numbers + 1000. + +Data is stored in the system user's configuration directory under +"mox-localserve", e.g. $HOME/.config/mox-localserve/ on linux, but can be +overridden with the -dir flag. If the directory does not yet exist, it is +automatically initialized with configuration files, an account with email +address mox@localhost and password moxmoxmox, and a newly generated self-signed +TLS certificate. + +All incoming email is accepted (if checks pass), unless the recipient localpart +ends with: + +- "temperror": fail with a temporary error code +- "permerror": fail with a permanent error code +- [45][0-9][0-9]: fail with the specific error code +- "timeout": no response (for an hour) + +If the localpart begins with "mailfrom" or "rcptto", the error is returned +during those commands instead of during "data". +` + golog.SetFlags(0) + + userConfDir, _ := os.UserConfigDir() + if userConfDir == "" { + userConfDir = "." + } + + var dir string + c.flag.StringVar(&dir, "dir", filepath.Join(userConfDir, "mox-localserve"), "configuration storage directory") + args := c.Parse() + if len(args) != 0 { + c.Usage() + } + + log := mlog.New("localserve") + + // Load config, creating a new one if needed. + if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { + err := writeLocalConfig(log, dir) + if err != nil { + log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir)) + } + } else if err != nil { + log.Fatalx("stat config dir", err, mlog.Field("dir", dir)) + } else if err := localLoadConfig(log, dir); err != nil { + log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, mlog.Field("dir", dir)) + } + + // Initialize receivedid. + recvidbuf, err := os.ReadFile(filepath.Join(dir, "receivedid.key")) + if err == nil && len(recvidbuf) != 16+8 { + err = fmt.Errorf("bad length %d, need 16+8", len(recvidbuf)) + } + if err != nil { + log.Errorx("reading receivedid.key", err) + recvidbuf = make([]byte, 16+8) + _, err := cryptorand.Read(recvidbuf) + if err != nil { + log.Fatalx("read random recvid key", err) + } + } + if err := mox.ReceivedIDInit(recvidbuf[:16], recvidbuf[16:]); err != nil { + log.Fatalx("init receivedid", err) + } + + // Make smtp server accept all email and deliver to account "mox". + smtpserver.Localserve = true + // Tell queue it shouldn't be queuing/delivering. + queue.Localserve = true + + mox.ListenImmediate = true + const mtastsdbRefresher = false + const skipForkExec = true + if err := start(mtastsdbRefresher, skipForkExec); err != nil { + log.Fatalx("starting mox", err) + } + golog.Printf("mox, version %s", moxvar.Version) + golog.Print("") + golog.Printf("the default user is mox@localhost, with password moxmoxmox") + golog.Printf("the default admin password is moxadmin") + golog.Printf("port numbers are those common for the services + 1000") + golog.Printf("tls uses generated self-signed certificate %s", filepath.Join(dir, "localhost.crt")) + golog.Printf("all incoming email is accepted (if checks pass), unless the recipient localpart ends with:") + golog.Print("") + golog.Printf(`- "temperror": fail with a temporary error code.`) + golog.Printf(`- "permerror": fail with a permanent error code.`) + golog.Printf(`- [45][0-9][0-9]: fail with the specific error code.`) + golog.Printf(`- "timeout": no response (for an hour).`) + golog.Print("") + golog.Printf(`if the localpart begins with "mailfrom" or "rcptto", the error is returned during those commands instead of during "data"`) + golog.Print("") + golog.Print(" smtp://localhost:1025 - receive email") + golog.Print("smtps://mox%40localhost:moxmoxmox@localhost:1465 - send email") + golog.Print(" smtp://mox%40localhost:moxmoxmox@localhost:1587 - send email (without tls)") + golog.Print("imaps://mox%40localhost:moxmoxmox@localhost:1993 - read email") + golog.Print(" imap://mox%40localhost:moxmoxmox@localhost:1143 - read email (without tls)") + golog.Print("https://mox%40localhost:moxmoxmox@localhost:1443 - account https") + golog.Print(" http://mox%40localhost:moxmoxmox@localhost:1080 - account http (without tls)") + golog.Print("https://admin:moxadmin@localhost:1443/admin/ - admin https") + golog.Print(" http://admin:moxadmin@localhost:1080/admin/ - admin http (without tls)") + golog.Print("") + golog.Printf("serving from %s", dir) + + ctlpath := mox.DataDirPath("ctl") + _ = os.Remove(ctlpath) + ctl, err := net.Listen("unix", ctlpath) + if err != nil { + log.Fatalx("listen on ctl unix domain socket", err) + } + go func() { + for { + conn, err := ctl.Accept() + if err != nil { + log.Printx("accept for ctl", err) + continue + } + cid := mox.Cid() + ctx := context.WithValue(mox.Context, mlog.CidKey, cid) + go servectl(ctx, log.WithCid(cid), conn, func() { shutdown(log) }) + } + }() + + // Graceful shutdown. + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) + sig := <-sigc + log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig)) + shutdown(log) + if num, ok := sig.(syscall.Signal); ok { + os.Exit(int(num)) + } else { + os.Exit(1) + } +} + +func writeLocalConfig(log *mlog.Log, dir string) (rerr error) { + defer func() { + x := recover() + if x != nil { + if err, ok := x.(error); ok { + rerr = err + } + } + if rerr != nil { + err := os.RemoveAll(dir) + log.Check(err, "removing config directory", mlog.Field("dir", dir)) + } + }() + + xcheck := func(err error, msg string) { + if err != nil { + panic(fmt.Errorf("%s: %s", msg, err)) + } + } + + os.MkdirAll(dir, 0770) + + // Generate key and self-signed certificate for use with TLS. + privKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + xcheck(err, "generating ecdsa key for self-signed certificate") + privKeyDER, err := x509.MarshalPKCS8PrivateKey(privKey) + xcheck(err, "marshal private key to pkcs8") + privBlock := &pem.Block{ + Type: "PRIVATE KEY", + Headers: map[string]string{ + "Note": "ECDSA key generated by mox localserve for self-signed certificate.", + }, + Bytes: privKeyDER, + } + var privPEM bytes.Buffer + err = pem.Encode(&privPEM, privBlock) + xcheck(err, "pem-encoding private key") + err = os.WriteFile(filepath.Join(dir, "localhost.key"), privPEM.Bytes(), 0660) + xcheck(err, "writing private key for self-signed certificate") + + // Now the certificate. + template := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().Unix()), // Required field. + DNSNames: []string{"localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(4 * 365 * 24 * time.Hour), + Issuer: pkix.Name{ + Organization: []string{"mox localserve"}, + }, + Subject: pkix.Name{ + Organization: []string{"mox localserve"}, + CommonName: "localhost", + }, + } + certDER, err := x509.CreateCertificate(cryptorand.Reader, template, template, privKey.Public(), privKey) + xcheck(err, "making self-signed certificate") + + pubBlock := &pem.Block{ + Type: "CERTIFICATE", + // Comments (header) would cause failure to parse the certificate when we load the config. + Bytes: certDER, + } + var crtPEM bytes.Buffer + err = pem.Encode(&crtPEM, pubBlock) + xcheck(err, "pem-encoding self-signed certificate") + err = os.WriteFile(filepath.Join(dir, "localhost.crt"), crtPEM.Bytes(), 0660) + xcheck(err, "writing self-signed certificate") + + // Write adminpasswd. + adminpw := "moxadmin" + adminpwhash, err := bcrypt.GenerateFromPassword([]byte(adminpw), bcrypt.DefaultCost) + xcheck(err, "generating hash for admin password") + err = os.WriteFile(filepath.Join(dir, "adminpasswd"), adminpwhash, 0660) + xcheck(err, "writing adminpasswd file") + + // Write mox.conf. + + local := config.Listener{ + IPs: []string{"127.0.0.1", "::1"}, + TLS: &config.TLS{ + KeyCerts: []config.KeyCert{ + { + CertFile: "localhost.crt", + KeyFile: "localhost.key", + }, + }, + }, + } + local.SMTP.Enabled = true + local.SMTP.Port = 1025 + local.Submission.Enabled = true + local.Submission.Port = 1587 + local.Submission.NoRequireSTARTTLS = true + local.Submissions.Enabled = true + local.Submissions.Port = 1465 + local.IMAP.Enabled = true + local.IMAP.Port = 1143 + local.IMAP.NoRequireSTARTTLS = true + local.IMAPS.Enabled = true + local.IMAPS.Port = 1993 + local.AccountHTTP.Enabled = true + local.AccountHTTP.Port = 1080 + local.AccountHTTPS.Enabled = true + local.AccountHTTPS.Port = 1443 + local.AdminHTTP.Enabled = true + local.AdminHTTP.Port = 1080 + local.AdminHTTPS.Enabled = true + local.AdminHTTPS.Port = 1443 + local.MetricsHTTP.Enabled = true + local.MetricsHTTP.Port = 1081 + local.WebserverHTTP.Enabled = true + local.WebserverHTTP.Port = 1080 + local.WebserverHTTPS.Enabled = true + local.WebserverHTTPS.Port = 1443 + + static := config.Static{ + DataDir: ".", + LogLevel: "traceauth", + Hostname: "localhost", + User: fmt.Sprintf("%d", os.Getuid()), + AdminPasswordFile: "adminpasswd", + Listeners: map[string]config.Listener{ + "local": local, + }, + } + tlsca := struct { + AdditionalToSystem bool `sconf:"optional"` + CertFiles []string `sconf:"optional"` + }{true, []string{"localhost.crt"}} + static.TLS.CA = &tlsca + static.Postmaster.Account = "mox" + static.Postmaster.Mailbox = "Inbox" + + var moxconfBuf bytes.Buffer + err = sconf.WriteDocs(&moxconfBuf, static) + xcheck(err, "making mox.conf") + + err = os.WriteFile(filepath.Join(dir, "mox.conf"), moxconfBuf.Bytes(), 0660) + xcheck(err, "writing mox.conf") + + // Write domains.conf. + acc := config.Account{ + RejectsMailbox: "Rejects", + Destinations: map[string]config.Destination{ + "mox@localhost": {}, + }, + } + acc.AutomaticJunkFlags.Enabled = true + acc.AutomaticJunkFlags.JunkMailboxRegexp = "^(junk|spam)" + acc.AutomaticJunkFlags.NeutralMailboxRegexp = "^(inbox|neutral|postmaster|dmarc|tlsrpt|rejects)" + acc.JunkFilter = &config.JunkFilter{ + Threshold: 0.95, + Params: junk.Params{ + Onegrams: true, + MaxPower: .01, + TopWords: 10, + IgnoreWords: .1, + RareWords: 2, + }, + } + + dynamic := config.Dynamic{ + Domains: map[string]config.Domain{ + "localhost": { + LocalpartCatchallSeparator: "+", + }, + }, + Accounts: map[string]config.Account{ + "mox": acc, + }, + WebHandlers: []config.WebHandler{ + { + LogName: "workdir", + Domain: "localhost", + PathRegexp: "^/workdir/", + DontRedirectPlainHTTP: true, + WebStatic: &config.WebStatic{ + StripPrefix: "/workdir/", + Root: ".", + ListFiles: true, + }, + }, + }, + } + var domainsconfBuf bytes.Buffer + err = sconf.WriteDocs(&domainsconfBuf, dynamic) + xcheck(err, "making domains.conf") + + err = os.WriteFile(filepath.Join(dir, "domains.conf"), domainsconfBuf.Bytes(), 0660) + xcheck(err, "writing domains.conf") + + // Write receivedid.key. + recvidbuf := make([]byte, 16+8) + _, err = cryptorand.Read(recvidbuf) + xcheck(err, "reading random recvid data") + err = os.WriteFile(filepath.Join(dir, "receivedid.key"), recvidbuf, 0660) + xcheck(err, "writing receivedid.key") + + // Load config, so we can access the account. + err = localLoadConfig(log, dir) + xcheck(err, "loading config") + + // Set password on account. + a, _, err := store.OpenEmail("mox@localhost") + xcheck(err, "opening account to set password") + password := "moxmoxmox" + err = a.SetPassword(password) + xcheck(err, "setting password") + err = a.Close() + xcheck(err, "closing account") + + golog.Printf("config created in %s", dir) + return nil +} + +func localLoadConfig(log *mlog.Log, dir string) error { + mox.ConfigStaticPath = filepath.Join(dir, "mox.conf") + mox.ConfigDynamicPath = filepath.Join(dir, "domains.conf") + errs := mox.LoadConfig(context.Background(), false) + if len(errs) > 1 { + log.Error("loading config generated config file: multiple errors") + for _, err := range errs { + log.Errorx("config error", err) + } + return fmt.Errorf("stopping after multiple config errors") + } else if len(errs) == 1 { + return fmt.Errorf("loading config file: %v", errs[0]) + } + return nil +} diff --git a/main.go b/main.go index 3fcf0f4..3afc2d2 100644 --- a/main.go +++ b/main.go @@ -90,6 +90,7 @@ var commands = []struct { {"import mbox", cmdImportMbox}, {"export maildir", cmdExportMaildir}, {"export mbox", cmdExportMbox}, + {"localserve", cmdLocalserve}, {"help", cmdHelp}, {"config test", cmdConfigTest}, diff --git a/mox-/config.go b/mox-/config.go index ddd914d..d4dd7c7 100644 --- a/mox-/config.go +++ b/mox-/config.go @@ -313,9 +313,6 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error { // MustLoadConfig loads the config, quitting on errors. func MustLoadConfig(checkACMEHosts bool) { - Shutdown, ShutdownCancel = context.WithCancel(context.Background()) - Context, ContextCancel = context.WithCancel(context.Background()) - errs := LoadConfig(context.Background(), checkACMEHosts) if len(errs) > 1 { xlog.Error("loading config file: multiple errors") @@ -331,6 +328,9 @@ func MustLoadConfig(checkACMEHosts bool) { // LoadConfig attempts to parse and load a config, returning any errors // encountered. func LoadConfig(ctx context.Context, checkACMEHosts bool) []error { + Shutdown, ShutdownCancel = context.WithCancel(context.Background()) + Context, ContextCancel = context.WithCancel(context.Background()) + c, errs := ParseConfig(ctx, ConfigStaticPath, false, false, checkACMEHosts) if len(errs) > 0 { return errs diff --git a/mox-/lifecycle.go b/mox-/lifecycle.go index e22f7ea..2065e47 100644 --- a/mox-/lifecycle.go +++ b/mox-/lifecycle.go @@ -111,11 +111,15 @@ func CleanupPassedSockets() { } } +// Make Listen listen immediately, regardless of running as root or other user, in +// case ForkExecUnprivileged is not used. +var ListenImmediate bool + // Listen returns a newly created network listener when starting as root, and // otherwise (not root) returns a network listener from a file descriptor that was // passed by the parent root process. func Listen(network, addr string) (net.Listener, error) { - if os.Getuid() != 0 { + if os.Getuid() != 0 && !ListenImmediate { f, ok := listens[addr] if !ok { return nil, fmt.Errorf("no file descriptor for listener %s", addr) diff --git a/queue/queue.go b/queue/queue.go index 23a2d85..de4b751 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -72,6 +72,9 @@ var jitter = mox.NewRand() var queueDB *bstore.DB +// Set for mox localserve, to prevent queueing. +var Localserve bool + // Msg is a message in the queue. type Msg struct { ID int64 @@ -179,6 +182,11 @@ func Count() (int, error) { func Add(log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) error { // todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759 + if Localserve { + // Safety measure, shouldn't happen. + return fmt.Errorf("no queuing with localserve") + } + tx, err := queueDB.Begin(true) if err != nil { return fmt.Errorf("begin transaction: %w", err) diff --git a/serve.go b/serve.go index 7d03698..40dc5f1 100644 --- a/serve.go +++ b/serve.go @@ -127,6 +127,7 @@ func monitorDNSBL(log *mlog.Log) { } } +// also see localserve.go, code is similar or even shared. func cmdServe(c *cmd) { c.help = `Start mox, serving SMTP/IMAP/HTTPS. @@ -320,38 +321,6 @@ requested, other TLS certificates are requested on demand. go monitorDNSBL(log) - shutdown := func() { - // We indicate we are shutting down. Causes new connections and new SMTP commands - // to be rejected. Should stop active connections pretty quickly. - mox.ShutdownCancel() - - // Now we are going to wait for all connections to be gone, up to a timeout. - done := mox.Connections.Done() - second := time.Tick(time.Second) - select { - case <-done: - log.Print("connections shutdown, waiting until 1 second passed") - <-second - - case <-time.Tick(3 * time.Second): - // We now cancel all pending operations, and set an immediate deadline on sockets. - // Should get us a clean shutdown relatively quickly. - mox.ContextCancel() - mox.Connections.Shutdown() - - second := time.Tick(time.Second) - select { - case <-done: - log.Print("no more connections, shutdown is clean, waiting until 1 second passed") - <-second // Still wait for second, giving processes like imports a chance to clean up. - case <-second: - log.Print("shutting down with pending sockets") - } - } - err := os.Remove(mox.DataDirPath("ctl")) - log.Check(err, "removing ctl unix domain socket during shutdown") - } - ctlpath := mox.DataDirPath("ctl") _ = os.Remove(ctlpath) ctl, err := net.Listen("unix", ctlpath) @@ -367,7 +336,7 @@ requested, other TLS certificates are requested on demand. } cid := mox.Cid() ctx := context.WithValue(mox.Context, mlog.CidKey, cid) - go servectl(ctx, log.WithCid(cid), conn, shutdown) + go servectl(ctx, log.WithCid(cid), conn, func() { shutdown(log) }) } }() @@ -398,7 +367,7 @@ requested, other TLS certificates are requested on demand. signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) sig := <-sigc log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig)) - shutdown() + shutdown(log) if num, ok := sig.(syscall.Signal); ok { os.Exit(int(num)) } else { @@ -406,6 +375,38 @@ requested, other TLS certificates are requested on demand. } } +func shutdown(log *mlog.Log) { + // We indicate we are shutting down. Causes new connections and new SMTP commands + // to be rejected. Should stop active connections pretty quickly. + mox.ShutdownCancel() + + // Now we are going to wait for all connections to be gone, up to a timeout. + done := mox.Connections.Done() + second := time.Tick(time.Second) + select { + case <-done: + log.Print("connections shutdown, waiting until 1 second passed") + <-second + + case <-time.Tick(3 * time.Second): + // We now cancel all pending operations, and set an immediate deadline on sockets. + // Should get us a clean shutdown relatively quickly. + mox.ContextCancel() + mox.Connections.Shutdown() + + second := time.Tick(time.Second) + select { + case <-done: + log.Print("no more connections, shutdown is clean, waiting until 1 second passed") + <-second // Still wait for second, giving processes like imports a chance to clean up. + case <-second: + log.Print("shutting down with pending sockets") + } + } + err := os.Remove(mox.DataDirPath("ctl")) + log.Check(err, "removing ctl unix domain socket during shutdown") +} + // Set correct permissions for mox working directory, binary, config and data and service file. // // We require being able to stat the basic non-optional paths. Then we'll try to diff --git a/smtpserver/reputation_test.go b/smtpserver/reputation_test.go index 32245ad..927a07d 100644 --- a/smtpserver/reputation_test.go +++ b/smtpserver/reputation_test.go @@ -55,17 +55,7 @@ func TestReputation(t *testing.T) { } } - var ipmasked1, ipmasked2, ipmasked3 string - var xip = net.ParseIP(ip) - if xip.To4() != nil { - ipmasked1 = xip.String() - ipmasked2 = xip.Mask(net.CIDRMask(26, 32)).String() - ipmasked3 = xip.Mask(net.CIDRMask(21, 32)).String() - } else { - ipmasked1 = xip.Mask(net.CIDRMask(64, 128)).String() - ipmasked2 = xip.Mask(net.CIDRMask(48, 128)).String() - ipmasked3 = xip.Mask(net.CIDRMask(32, 128)).String() - } + ipmasked1, ipmasked2, ipmasked3 := ipmasked(net.ParseIP(ip)) uidgen++ m := store.Message{ diff --git a/smtpserver/server.go b/smtpserver/server.go index c69be90..a046827 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -19,6 +19,7 @@ import ( "net" "os" "runtime/debug" + "strconv" "strings" "sync" "time" @@ -61,6 +62,10 @@ var xlog = mlog.New("smtpserver") // These errors signal the connection must be closed. var errIO = errors.New("fatal io error") +// If set, regular delivery/submit is sidestepped, email is accepted and +// delivered to the account named mox. +var Localserve bool + var limiterConnectionRate, limiterConnections *ratelimit.Limiter // For delivery rate limiting. Variable because changed during tests. @@ -1287,6 +1292,11 @@ func (c *conn) cmdMail(p *parser) { c.log.Info("delivery from address without domain", mlog.Field("mailfrom", rpath.String())) xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7Other0, "domain name required") } + + if Localserve && strings.HasPrefix(string(rpath.Localpart), "mailfrom") { + c.xlocalserveError(rpath.Localpart) + } + c.mailFrom = &rpath c.bwritecodeline(smtp.C250Completed, smtp.SeAddr1Other0, "looking good", nil) @@ -1347,7 +1357,7 @@ func (c *conn) cmdRcpt(p *parser) { // ../rfc/5321:3598 // ../rfc/5321:4045 // Also see ../rfc/7489:2214 - if !c.submission && len(c.recipients) == 1 { + if !c.submission && len(c.recipients) == 1 && !Localserve { // note: because of check above, mailFrom cannot be the null address. var pass bool d := c.mailFrom.IPDomain.Domain @@ -1376,7 +1386,18 @@ func (c *conn) cmdRcpt(p *parser) { } } - if len(fpath.IPDomain.IP) > 0 { + if Localserve { + if strings.HasPrefix(string(fpath.Localpart), "rcptto") { + c.xlocalserveError(fpath.Localpart) + } + + // If account or destination doesn't exist, it will be handled during delivery. For + // submissions, which is the common case, we'll deliver to the logged in user, + // which is typically the mox user. + acc, _ := mox.Conf.Account("mox") + dest := acc.Destinations["mox@localhost"] + c.recipients = append(c.recipients, rcptAccount{fpath, true, "mox", dest, "mox@localhost"}) + } else if len(fpath.IPDomain.IP) > 0 { if !c.submission { xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SeAddr1UnknownDestMailbox1, "not accepting email for ip") } @@ -1674,23 +1695,95 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr } msgPrefix = append(msgPrefix, []byte(authResults.Header())...) - for i, rcptAcc := range c.recipients { - xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...) - // todo: don't convert the headers to a body? it seems the body part is optional. does this have consequences for us in other places? ../rfc/5322:343 - if !msgWriter.HaveHeaders { - xmsgPrefix = append(xmsgPrefix, "\r\n"...) + if Localserve { + var timeout bool + c.account.WithWLock(func() { + for i, rcptAcc := range c.recipients { + var code int + code, timeout = localserveNeedsError(rcptAcc.rcptTo.Localpart) + if timeout { + // Get out of wlock, and sleep there. + return + } else if code != 0 { + c.log.Info("failure due to special localpart", mlog.Field("code", code)) + xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code) + } + + xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...) + // todo: don't convert the headers to a body? it seems the body part is optional. does this have consequences for us in other places? ../rfc/5322:343 + if !msgWriter.HaveHeaders { + xmsgPrefix = append(xmsgPrefix, "\r\n"...) + } + msgSize := int64(len(xmsgPrefix)) + msgWriter.Size + + ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP) + m := store.Message{ + Received: time.Now(), + RemoteIP: c.remoteIP.String(), + RemoteIPMasked1: ipmasked1, + RemoteIPMasked2: ipmasked2, + RemoteIPMasked3: ipmasked3, + EHLODomain: c.hello.Domain.Name(), + MailFrom: c.mailFrom.String(), + MailFromLocalpart: c.mailFrom.Localpart, + MailFromDomain: c.mailFrom.IPDomain.Domain.Name(), + RcptToLocalpart: rcptAcc.rcptTo.Localpart, + RcptToDomain: rcptAcc.rcptTo.IPDomain.Domain.Name(), + MsgFromLocalpart: msgFrom.Localpart, + MsgFromDomain: msgFrom.Domain.Name(), + MsgFromOrgDomain: publicsuffix.Lookup(ctx, msgFrom.Domain).Name(), + EHLOValidated: true, + MailFromValidated: true, + MsgFromValidated: true, + EHLOValidation: store.ValidationPass, + MailFromValidation: store.ValidationRelaxed, + MsgFromValidation: store.ValidationRelaxed, + DKIMDomains: nil, + Size: msgSize, + MsgPrefix: xmsgPrefix, + } + + if err := c.account.Deliver(c.log, rcptAcc.destination, &m, dataFile, i == len(c.recipients)-1); err != nil { + // Aborting the transaction is not great. But continuing and generating DSNs will + // probably result in errors as well... + metricSubmission.WithLabelValues("localserveerror").Inc() + c.log.Errorx("delivering message", err) + xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err) + } + metricSubmission.WithLabelValues("ok").Inc() + c.log.Info("submitted message delivered", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize)) + } + }) + + if timeout { + c.log.Info("timing out submission due to special localpart") + mox.Sleep(mox.Context, time.Hour) + xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out submission due to special localpart") } - msgSize := int64(len(xmsgPrefix)) + msgWriter.Size - if err := queue.Add(c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, xmsgPrefix, dataFile, nil, i == len(c.recipients)-1); err != nil { - // Aborting the transaction is not great. But continuing and generating DSNs will - // probably result in errors as well... - metricSubmission.WithLabelValues("queueerror").Inc() - c.log.Errorx("queuing message", err) - xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err) + } else { + // We always deliver through the queue. It would be more efficient to deliver + // directly, but we don't want to circumvent all the anti-spam measures. Accounts + // on a single mox instance should be allowed to block each other. + + for i, rcptAcc := range c.recipients { + xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...) + // todo: don't convert the headers to a body? it seems the body part is optional. does this have consequences for us in other places? ../rfc/5322:343 + if !msgWriter.HaveHeaders { + xmsgPrefix = append(xmsgPrefix, "\r\n"...) + } + + msgSize := int64(len(xmsgPrefix)) + msgWriter.Size + if err := queue.Add(c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, xmsgPrefix, dataFile, nil, i == len(c.recipients)-1); err != nil { + // Aborting the transaction is not great. But continuing and generating DSNs will + // probably result in errors as well... + metricSubmission.WithLabelValues("queueerror").Inc() + c.log.Errorx("queuing message", err) + xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err) + } + metricSubmission.WithLabelValues("ok").Inc() + c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize)) } - metricSubmission.WithLabelValues("ok").Inc() - c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize)) } err = dataFile.Close() c.log.Check(err, "closing file after submission") @@ -1703,6 +1796,55 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr c.writecodeline(smtp.C250Completed, smtp.SeMailbox2Other0, "it is done", nil) } +func ipmasked(ip net.IP) (string, string, string) { + if ip.To4() != nil { + m1 := ip.String() + m2 := ip.Mask(net.CIDRMask(26, 32)).String() + m3 := ip.Mask(net.CIDRMask(21, 32)).String() + return m1, m2, m3 + } + m1 := ip.Mask(net.CIDRMask(64, 128)).String() + m2 := ip.Mask(net.CIDRMask(48, 128)).String() + m3 := ip.Mask(net.CIDRMask(32, 128)).String() + return m1, m2, m3 +} + +func localserveNeedsError(lp smtp.Localpart) (code int, timeout bool) { + s := string(lp) + if strings.HasSuffix(s, "temperror") { + return smtp.C451LocalErr, false + } else if strings.HasSuffix(s, "permerror") { + return smtp.C550MailboxUnavail, false + } else if strings.HasSuffix(s, "timeout") { + return 0, true + } + if len(s) < 3 { + return 0, false + } + s = s[len(s)-3:] + v, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return 0, false + } + if v < 400 || v > 600 { + return 0, false + } + return int(v), false +} + +func (c *conn) xlocalserveError(lp smtp.Localpart) { + code, timeout := localserveNeedsError(lp) + if timeout { + c.log.Info("timing out due to special localpart") + mox.Sleep(mox.Context, time.Hour) + xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out command due to special localpart") + } else if code != 0 { + c.log.Info("failure due to special localpart", mlog.Field("code", code)) + metricDelivery.WithLabelValues("delivererror", "localserve").Inc() + xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code) + } +} + // deliver is called for incoming messages from external, typically untrusted // sources. i.e. not submitted by authenticated users. func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, pdataFile **os.File) { @@ -1968,16 +2110,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain)) // Prepare for analyzing content, calculating reputation. - var ipmasked1, ipmasked2, ipmasked3 string - if c.remoteIP.To4() != nil { - ipmasked1 = c.remoteIP.String() - ipmasked2 = c.remoteIP.Mask(net.CIDRMask(26, 32)).String() - ipmasked3 = c.remoteIP.Mask(net.CIDRMask(21, 32)).String() - } else { - ipmasked1 = c.remoteIP.Mask(net.CIDRMask(64, 128)).String() - ipmasked2 = c.remoteIP.Mask(net.CIDRMask(48, 128)).String() - ipmasked3 = c.remoteIP.Mask(net.CIDRMask(32, 128)).String() - } + ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP) var verifiedDKIMDomains []string for _, r := range dkimResults { // A message can have multiple signatures for the same identity. For example when @@ -2019,7 +2152,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // For each recipient, do final spam analysis and delivery. for _, rcptAcc := range c.recipients { - log := c.log.Fields(mlog.Field("mailfrom", c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo)) // If this is not a valid local user, we send back a DSN. This can only happen when @@ -2238,34 +2370,47 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW mox.Sleep(mox.Context, reputationlessSenderDeliveryDelay) } - acc.WithWLock(func() { - // Gather the message-id before we deliver and the file may be consumed. - if !parsedMessageID { - if p, err := message.Parse(store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil { - log.Infox("parsing message for message-id", err) - } else if header, err := p.Header(); err != nil { - log.Infox("parsing message header for message-id", err) - } else { - messageID = header.Get("Message-Id") - } + // Gather the message-id before we deliver and the file may be consumed. + if !parsedMessageID { + if p, err := message.Parse(store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil { + log.Infox("parsing message for message-id", err) + } else if header, err := p.Header(); err != nil { + log.Infox("parsing message header for message-id", err) + } else { + messageID = header.Get("Message-Id") } + } - if err := acc.Deliver(log, rcptAcc.destination, m, dataFile, false); err != nil { - log.Errorx("delivering", err) - metricDelivery.WithLabelValues("delivererror", a.reason).Inc() - addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing") - return + if Localserve { + code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart) + if timeout { + c.log.Info("timing out due to special localpart") + mox.Sleep(mox.Context, time.Hour) + xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeOther00}, "timing out delivery due to special localpart") + } else if code != 0 { + c.log.Info("failure due to special localpart", mlog.Field("code", code)) + metricDelivery.WithLabelValues("delivererror", "localserve").Inc() + addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code)) } - metricDelivery.WithLabelValues("delivered", a.reason).Inc() - log.Info("incoming message delivered", mlog.Field("reason", a.reason)) - - conf, _ := acc.Conf() - if conf.RejectsMailbox != "" && messageID != "" { - if err := acc.RejectsRemove(log, conf.RejectsMailbox, messageID); err != nil { - log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID)) + } else { + acc.WithWLock(func() { + if err := acc.Deliver(log, rcptAcc.destination, m, dataFile, false); err != nil { + log.Errorx("delivering", err) + metricDelivery.WithLabelValues("delivererror", a.reason).Inc() + addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing") + return } - } - }) + metricDelivery.WithLabelValues("delivered", a.reason).Inc() + log.Info("incoming message delivered", mlog.Field("reason", a.reason)) + + conf, _ := acc.Conf() + if conf.RejectsMailbox != "" && messageID != "" { + if err := acc.RejectsRemove(log, conf.RejectsMailbox, messageID); err != nil { + log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID)) + } + } + }) + } err = acc.Close() log.Check(err, "closing account after delivering") @@ -2347,7 +2492,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } dsnMsg.Original = header - if err := queueDSN(c, *c.mailFrom, dsnMsg); err != nil { + if Localserve { + c.log.Error("not queueing dsn for incoming delivery due to localserve") + } else if err := queueDSN(c, *c.mailFrom, dsnMsg); err != nil { metricServerErrors.WithLabelValues("queuedsn").Inc() c.log.Errorx("queuing DSN for incoming delivery, no DSN sent", err) }