add "mox localserve" subcommand, for running mox locally for email-related testing/developing

localserve creates a config for listening on localhost for
imap/smtp/submission/http, on port numbers 1000 + the common service port
numbers. all incoming email is accepted (if checks pass), and a few pattern in
localparts are recognized and result in delivery errors.

README.md

@@ -34,6 +34,8 @@ See Quickstart below to get started.
 - Webserver with serving static files and forwarding requests (reverse
   proxy), so port 443 can also be used to serve websites.
 - Prometheus metrics and structured logging for operational insight.
+- "localserve" subcommand for running mox locally for email-related
+  testing/developing.
 
 Mox is available under the MIT-license and was created by Mechiel Lukkien,
 mechiel@ueber.net. Mox includes the Public Suffix List by Mozilla, under Mozilla
@@ -109,8 +111,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili
 
 - Strict vs lax mode, defaulting to lax when receiving from the internet, and
   strict when sending.
-- "developer server" mode, to easily launch a local SMTP/IMAP server to test
-  your apps mail sending capabilities.
 - Rate limiting and spam detection for submitted/outgoing messages, to reduce
   impact when an account gets compromised.
 - Privilege separation, isolating parts of the application to more restricted
@@ -118,7 +118,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili
 - DANE and DNSSEC.
 - Sending DMARC and TLS reports (currently only receiving).
 - OAUTH2 support, for single sign on.
-- ACME verification over HTTP (in addition to current tls-alpn01).
 - Add special IMAP mailbox ("Queue?") that contains queued but
   not-yet-delivered messages.
 - Sieve for filtering (for now see Rulesets in the account config)

doc.go

@@ -27,6 +27,7 @@ low-maintenance self-hosted email.
 	mox import mbox accountname mailboxname mbox
 	mox export maildir dst-dir account-path [mailbox]
 	mox export mbox dst-dir account-path [mailbox]
+	mox localserve
 	mox help [command ...]
 	mox config test
 	mox config dnscheck domain
@@ -291,6 +292,36 @@ otherwise reconstructing the original could lose a ">".
 
 	usage: mox export mbox dst-dir account-path [mailbox]
 
+# mox localserve
+
+Start a local SMTP/IMAP server that accepts all messages, useful when testing/developing software that sends email.
+
+Localserve starts mox with a configuration suitable for local email-related
+software development/testing. It listens for SMTP/Submission(s), IMAP(s) and
+HTTP(s), on the regular port numbers + 1000.
+
+Data is stored in the system user's configuration directory under
+"mox-localserve", e.g. $HOME/.config/mox-localserve/ on linux, but can be
+overridden with the -dir flag. If the directory does not yet exist, it is
+automatically initialized with configuration files, an account with email
+address mox@localhost and password moxmoxmox, and a newly generated self-signed
+TLS certificate.
+
+All incoming email is accepted (if checks pass), unless the recipient localpart
+ends with:
+
+- "temperror": fail with a temporary error code
+- "permerror": fail with a permanent error code
+- [45][0-9][0-9]: fail with the specific error code
+- "timeout": no response (for an hour)
+
+If the localpart begins with "mailfrom" or "rcptto", the error is returned
+during those commands instead of during "data".
+
+	usage: mox localserve
+	  -dir string
+	    	configuration storage directory (default "$userconfigdir/mox-localserve")
+
 # mox help
 
 Prints help about matching commands.

gendoc.sh

@@ -28,7 +28,9 @@ directory) through the -config flag or MOXCONF environment variable.
 
 EOF
 
-./mox helpall 2>&1
+# setting XDG_CONFIG_HOME ensures "mox localserve" has reasonable default
+# values in its help output.
+XDG_CONFIG_HOME='$userconfigdir' ./mox helpall 2>&1
 
 cat <<EOF
 */

integration_test.go

@@ -39,8 +39,6 @@ func tcheck(t *testing.T, err error, msg string) {
 // We check if we receive the message.
 func TestDeliver(t *testing.T) {
 	mlog.Logfmt = true
-	mox.Context, mox.ContextCancel = context.WithCancel(context.Background())
-	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
 
 	// Remove state.
 	os.RemoveAll("testdata/integration/data")
@@ -53,7 +51,7 @@ func TestDeliver(t *testing.T) {
 	// Load mox config.
 	mox.ConfigStaticPath = "testdata/integration/config/mox.conf"
 	filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf")
-	if errs := mox.LoadConfig(mox.Context, false); len(errs) > 0 {
+	if errs := mox.LoadConfig(context.Background(), false); len(errs) > 0 {
 		t.Fatalf("loading mox config: %v", errs)
 	}
 

localserve.go

@@ -0,0 +1,405 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	cryptorand "crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	golog "log"
+	"math/big"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"golang.org/x/crypto/bcrypt"
+
+	"github.com/mjl-/sconf"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/junk"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/queue"
+	"github.com/mjl-/mox/smtpserver"
+	"github.com/mjl-/mox/store"
+)
+
+func cmdLocalserve(c *cmd) {
+	c.help = `Start a local SMTP/IMAP server that accepts all messages, useful when testing/developing software that sends email.
+
+Localserve starts mox with a configuration suitable for local email-related
+software development/testing. It listens for SMTP/Submission(s), IMAP(s) and
+HTTP(s), on the regular port numbers + 1000.
+
+Data is stored in the system user's configuration directory under
+"mox-localserve", e.g. $HOME/.config/mox-localserve/ on linux, but can be
+overridden with the -dir flag. If the directory does not yet exist, it is
+automatically initialized with configuration files, an account with email
+address mox@localhost and password moxmoxmox, and a newly generated self-signed
+TLS certificate.
+
+All incoming email is accepted (if checks pass), unless the recipient localpart
+ends with:
+
+- "temperror": fail with a temporary error code
+- "permerror": fail with a permanent error code
+- [45][0-9][0-9]: fail with the specific error code
+- "timeout": no response (for an hour)
+
+If the localpart begins with "mailfrom" or "rcptto", the error is returned
+during those commands instead of during "data".
+`
+	golog.SetFlags(0)
+
+	userConfDir, _ := os.UserConfigDir()
+	if userConfDir == "" {
+		userConfDir = "."
+	}
+
+	var dir string
+	c.flag.StringVar(&dir, "dir", filepath.Join(userConfDir, "mox-localserve"), "configuration storage directory")
+	args := c.Parse()
+	if len(args) != 0 {
+		c.Usage()
+	}
+
+	log := mlog.New("localserve")
+
+	// Load config, creating a new one if needed.
+	if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
+		err := writeLocalConfig(log, dir)
+		if err != nil {
+			log.Fatalx("creating mox localserve config", err, mlog.Field("dir", dir))
+		}
+	} else if err != nil {
+		log.Fatalx("stat config dir", err, mlog.Field("dir", dir))
+	} else if err := localLoadConfig(log, dir); err != nil {
+		log.Fatalx("loading mox localserve config (hint: when creating a new config with -dir, the directory must not yet exist)", err, mlog.Field("dir", dir))
+	}
+
+	// Initialize receivedid.
+	recvidbuf, err := os.ReadFile(filepath.Join(dir, "receivedid.key"))
+	if err == nil && len(recvidbuf) != 16+8 {
+		err = fmt.Errorf("bad length %d, need 16+8", len(recvidbuf))
+	}
+	if err != nil {
+		log.Errorx("reading receivedid.key", err)
+		recvidbuf = make([]byte, 16+8)
+		_, err := cryptorand.Read(recvidbuf)
+		if err != nil {
+			log.Fatalx("read random recvid key", err)
+		}
+	}
+	if err := mox.ReceivedIDInit(recvidbuf[:16], recvidbuf[16:]); err != nil {
+		log.Fatalx("init receivedid", err)
+	}
+
+	// Make smtp server accept all email and deliver to account "mox".
+	smtpserver.Localserve = true
+	// Tell queue it shouldn't be queuing/delivering.
+	queue.Localserve = true
+
+	mox.ListenImmediate = true
+	const mtastsdbRefresher = false
+	const skipForkExec = true
+	if err := start(mtastsdbRefresher, skipForkExec); err != nil {
+		log.Fatalx("starting mox", err)
+	}
+	golog.Printf("mox, version %s", moxvar.Version)
+	golog.Print("")
+	golog.Printf("the default user is mox@localhost, with password moxmoxmox")
+	golog.Printf("the default admin password is moxadmin")
+	golog.Printf("port numbers are those common for the services + 1000")
+	golog.Printf("tls uses generated self-signed certificate %s", filepath.Join(dir, "localhost.crt"))
+	golog.Printf("all incoming email is accepted (if checks pass), unless the recipient localpart ends with:")
+	golog.Print("")
+	golog.Printf(`- "temperror": fail with a temporary error code.`)
+	golog.Printf(`- "permerror": fail with a permanent error code.`)
+	golog.Printf(`- [45][0-9][0-9]: fail with the specific error code.`)
+	golog.Printf(`- "timeout": no response (for an hour).`)
+	golog.Print("")
+	golog.Printf(`if the localpart begins with "mailfrom" or "rcptto", the error is returned during those commands instead of during "data"`)
+	golog.Print("")
+	golog.Print(" smtp://localhost:1025                            - receive email")
+	golog.Print("smtps://mox%40localhost:moxmoxmox@localhost:1465  - send email")
+	golog.Print(" smtp://mox%40localhost:moxmoxmox@localhost:1587  - send email (without tls)")
+	golog.Print("imaps://mox%40localhost:moxmoxmox@localhost:1993  - read email")
+	golog.Print(" imap://mox%40localhost:moxmoxmox@localhost:1143  - read email (without tls)")
+	golog.Print("https://mox%40localhost:moxmoxmox@localhost:1443  - account https")
+	golog.Print(" http://mox%40localhost:moxmoxmox@localhost:1080  - account http (without tls)")
+	golog.Print("https://admin:moxadmin@localhost:1443/admin/      - admin https")
+	golog.Print(" http://admin:moxadmin@localhost:1080/admin/      - admin http (without tls)")
+	golog.Print("")
+	golog.Printf("serving from %s", dir)
+
+	ctlpath := mox.DataDirPath("ctl")
+	_ = os.Remove(ctlpath)
+	ctl, err := net.Listen("unix", ctlpath)
+	if err != nil {
+		log.Fatalx("listen on ctl unix domain socket", err)
+	}
+	go func() {
+		for {
+			conn, err := ctl.Accept()
+			if err != nil {
+				log.Printx("accept for ctl", err)
+				continue
+			}
+			cid := mox.Cid()
+			ctx := context.WithValue(mox.Context, mlog.CidKey, cid)
+			go servectl(ctx, log.WithCid(cid), conn, func() { shutdown(log) })
+		}
+	}()
+
+	// Graceful shutdown.
+	sigc := make(chan os.Signal, 1)
+	signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
+	sig := <-sigc
+	log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig))
+	shutdown(log)
+	if num, ok := sig.(syscall.Signal); ok {
+		os.Exit(int(num))
+	} else {
+		os.Exit(1)
+	}
+}
+
+func writeLocalConfig(log *mlog.Log, dir string) (rerr error) {
+	defer func() {
+		x := recover()
+		if x != nil {
+			if err, ok := x.(error); ok {
+				rerr = err
+			}
+		}
+		if rerr != nil {
+			err := os.RemoveAll(dir)
+			log.Check(err, "removing config directory", mlog.Field("dir", dir))
+		}
+	}()
+
+	xcheck := func(err error, msg string) {
+		if err != nil {
+			panic(fmt.Errorf("%s: %s", msg, err))
+		}
+	}
+
+	os.MkdirAll(dir, 0770)
+
+	// Generate key and self-signed certificate for use with TLS.
+	privKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	xcheck(err, "generating ecdsa key for self-signed certificate")
+	privKeyDER, err := x509.MarshalPKCS8PrivateKey(privKey)
+	xcheck(err, "marshal private key to pkcs8")
+	privBlock := &pem.Block{
+		Type: "PRIVATE KEY",
+		Headers: map[string]string{
+			"Note": "ECDSA key generated by mox localserve for self-signed certificate.",
+		},
+		Bytes: privKeyDER,
+	}
+	var privPEM bytes.Buffer
+	err = pem.Encode(&privPEM, privBlock)
+	xcheck(err, "pem-encoding private key")
+	err = os.WriteFile(filepath.Join(dir, "localhost.key"), privPEM.Bytes(), 0660)
+	xcheck(err, "writing private key for self-signed certificate")
+
+	// Now the certificate.
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().Unix()), // Required field.
+		DNSNames:     []string{"localhost"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(4 * 365 * 24 * time.Hour),
+		Issuer: pkix.Name{
+			Organization: []string{"mox localserve"},
+		},
+		Subject: pkix.Name{
+			Organization: []string{"mox localserve"},
+			CommonName:   "localhost",
+		},
+	}
+	certDER, err := x509.CreateCertificate(cryptorand.Reader, template, template, privKey.Public(), privKey)
+	xcheck(err, "making self-signed certificate")
+
+	pubBlock := &pem.Block{
+		Type: "CERTIFICATE",
+		// Comments (header) would cause failure to parse the certificate when we load the config.
+		Bytes: certDER,
+	}
+	var crtPEM bytes.Buffer
+	err = pem.Encode(&crtPEM, pubBlock)
+	xcheck(err, "pem-encoding self-signed certificate")
+	err = os.WriteFile(filepath.Join(dir, "localhost.crt"), crtPEM.Bytes(), 0660)
+	xcheck(err, "writing self-signed certificate")
+
+	// Write adminpasswd.
+	adminpw := "moxadmin"
+	adminpwhash, err := bcrypt.GenerateFromPassword([]byte(adminpw), bcrypt.DefaultCost)
+	xcheck(err, "generating hash for admin password")
+	err = os.WriteFile(filepath.Join(dir, "adminpasswd"), adminpwhash, 0660)
+	xcheck(err, "writing adminpasswd file")
+
+	// Write mox.conf.
+
+	local := config.Listener{
+		IPs: []string{"127.0.0.1", "::1"},
+		TLS: &config.TLS{
+			KeyCerts: []config.KeyCert{
+				{
+					CertFile: "localhost.crt",
+					KeyFile:  "localhost.key",
+				},
+			},
+		},
+	}
+	local.SMTP.Enabled = true
+	local.SMTP.Port = 1025
+	local.Submission.Enabled = true
+	local.Submission.Port = 1587
+	local.Submission.NoRequireSTARTTLS = true
+	local.Submissions.Enabled = true
+	local.Submissions.Port = 1465
+	local.IMAP.Enabled = true
+	local.IMAP.Port = 1143
+	local.IMAP.NoRequireSTARTTLS = true
+	local.IMAPS.Enabled = true
+	local.IMAPS.Port = 1993
+	local.AccountHTTP.Enabled = true
+	local.AccountHTTP.Port = 1080
+	local.AccountHTTPS.Enabled = true
+	local.AccountHTTPS.Port = 1443
+	local.AdminHTTP.Enabled = true
+	local.AdminHTTP.Port = 1080
+	local.AdminHTTPS.Enabled = true
+	local.AdminHTTPS.Port = 1443
+	local.MetricsHTTP.Enabled = true
+	local.MetricsHTTP.Port = 1081
+	local.WebserverHTTP.Enabled = true
+	local.WebserverHTTP.Port = 1080
+	local.WebserverHTTPS.Enabled = true
+	local.WebserverHTTPS.Port = 1443
+
+	static := config.Static{
+		DataDir:           ".",
+		LogLevel:          "traceauth",
+		Hostname:          "localhost",
+		User:              fmt.Sprintf("%d", os.Getuid()),
+		AdminPasswordFile: "adminpasswd",
+		Listeners: map[string]config.Listener{
+			"local": local,
+		},
+	}
+	tlsca := struct {
+		AdditionalToSystem bool     `sconf:"optional"`
+		CertFiles          []string `sconf:"optional"`
+	}{true, []string{"localhost.crt"}}
+	static.TLS.CA = &tlsca
+	static.Postmaster.Account = "mox"
+	static.Postmaster.Mailbox = "Inbox"
+
+	var moxconfBuf bytes.Buffer
+	err = sconf.WriteDocs(&moxconfBuf, static)
+	xcheck(err, "making mox.conf")
+
+	err = os.WriteFile(filepath.Join(dir, "mox.conf"), moxconfBuf.Bytes(), 0660)
+	xcheck(err, "writing mox.conf")
+
+	// Write domains.conf.
+	acc := config.Account{
+		RejectsMailbox: "Rejects",
+		Destinations: map[string]config.Destination{
+			"mox@localhost": {},
+		},
+	}
+	acc.AutomaticJunkFlags.Enabled = true
+	acc.AutomaticJunkFlags.JunkMailboxRegexp = "^(junk|spam)"
+	acc.AutomaticJunkFlags.NeutralMailboxRegexp = "^(inbox|neutral|postmaster|dmarc|tlsrpt|rejects)"
+	acc.JunkFilter = &config.JunkFilter{
+		Threshold: 0.95,
+		Params: junk.Params{
+			Onegrams:    true,
+			MaxPower:    .01,
+			TopWords:    10,
+			IgnoreWords: .1,
+			RareWords:   2,
+		},
+	}
+
+	dynamic := config.Dynamic{
+		Domains: map[string]config.Domain{
+			"localhost": {
+				LocalpartCatchallSeparator: "+",
+			},
+		},
+		Accounts: map[string]config.Account{
+			"mox": acc,
+		},
+		WebHandlers: []config.WebHandler{
+			{
+				LogName:               "workdir",
+				Domain:                "localhost",
+				PathRegexp:            "^/workdir/",
+				DontRedirectPlainHTTP: true,
+				WebStatic: &config.WebStatic{
+					StripPrefix: "/workdir/",
+					Root:        ".",
+					ListFiles:   true,
+				},
+			},
+		},
+	}
+	var domainsconfBuf bytes.Buffer
+	err = sconf.WriteDocs(&domainsconfBuf, dynamic)
+	xcheck(err, "making domains.conf")
+
+	err = os.WriteFile(filepath.Join(dir, "domains.conf"), domainsconfBuf.Bytes(), 0660)
+	xcheck(err, "writing domains.conf")
+
+	// Write receivedid.key.
+	recvidbuf := make([]byte, 16+8)
+	_, err = cryptorand.Read(recvidbuf)
+	xcheck(err, "reading random recvid data")
+	err = os.WriteFile(filepath.Join(dir, "receivedid.key"), recvidbuf, 0660)
+	xcheck(err, "writing receivedid.key")
+
+	// Load config, so we can access the account.
+	err = localLoadConfig(log, dir)
+	xcheck(err, "loading config")
+
+	// Set password on account.
+	a, _, err := store.OpenEmail("mox@localhost")
+	xcheck(err, "opening account to set password")
+	password := "moxmoxmox"
+	err = a.SetPassword(password)
+	xcheck(err, "setting password")
+	err = a.Close()
+	xcheck(err, "closing account")
+
+	golog.Printf("config created in %s", dir)
+	return nil
+}
+
+func localLoadConfig(log *mlog.Log, dir string) error {
+	mox.ConfigStaticPath = filepath.Join(dir, "mox.conf")
+	mox.ConfigDynamicPath = filepath.Join(dir, "domains.conf")
+	errs := mox.LoadConfig(context.Background(), false)
+	if len(errs) > 1 {
+		log.Error("loading config generated config file: multiple errors")
+		for _, err := range errs {
+			log.Errorx("config error", err)
+		}
+		return fmt.Errorf("stopping after multiple config errors")
+	} else if len(errs) == 1 {
+		return fmt.Errorf("loading config file: %v", errs[0])
+	}
+	return nil
+}

main.go

@@ -90,6 +90,7 @@ var commands = []struct {
 	{"import mbox", cmdImportMbox},
 	{"export maildir", cmdExportMaildir},
 	{"export mbox", cmdExportMbox},
+	{"localserve", cmdLocalserve},
 	{"help", cmdHelp},
 
 	{"config test", cmdConfigTest},

mox-/config.go

@@ -313,9 +313,6 @@ func writeDynamic(ctx context.Context, log *mlog.Log, c config.Dynamic) error {
 
 // MustLoadConfig loads the config, quitting on errors.
 func MustLoadConfig(checkACMEHosts bool) {
-	Shutdown, ShutdownCancel = context.WithCancel(context.Background())
-	Context, ContextCancel = context.WithCancel(context.Background())
-
 	errs := LoadConfig(context.Background(), checkACMEHosts)
 	if len(errs) > 1 {
 		xlog.Error("loading config file: multiple errors")
@@ -331,6 +328,9 @@ func MustLoadConfig(checkACMEHosts bool) {
 // LoadConfig attempts to parse and load a config, returning any errors
 // encountered.
 func LoadConfig(ctx context.Context, checkACMEHosts bool) []error {
+	Shutdown, ShutdownCancel = context.WithCancel(context.Background())
+	Context, ContextCancel = context.WithCancel(context.Background())
+
 	c, errs := ParseConfig(ctx, ConfigStaticPath, false, false, checkACMEHosts)
 	if len(errs) > 0 {
 		return errs

mox-/lifecycle.go

@@ -111,11 +111,15 @@ func CleanupPassedSockets() {
 	}
 }
 
+// Make Listen listen immediately, regardless of running as root or other user, in
+// case ForkExecUnprivileged is not used.
+var ListenImmediate bool
+
 // Listen returns a newly created network listener when starting as root, and
 // otherwise (not root) returns a network listener from a file descriptor that was
 // passed by the parent root process.
 func Listen(network, addr string) (net.Listener, error) {
-	if os.Getuid() != 0 {
+	if os.Getuid() != 0 && !ListenImmediate {
 		f, ok := listens[addr]
 		if !ok {
 			return nil, fmt.Errorf("no file descriptor for listener %s", addr)

queue/queue.go

@@ -72,6 +72,9 @@ var jitter = mox.NewRand()
 
 var queueDB *bstore.DB
 
+// Set for mox localserve, to prevent queueing.
+var Localserve bool
+
 // Msg is a message in the queue.
 type Msg struct {
 	ID                 int64
@@ -179,6 +182,11 @@ func Count() (int, error) {
 func Add(log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) error {
 	// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
 
+	if Localserve {
+		// Safety measure, shouldn't happen.
+		return fmt.Errorf("no queuing with localserve")
+	}
+
 	tx, err := queueDB.Begin(true)
 	if err != nil {
 		return fmt.Errorf("begin transaction: %w", err)

serve.go

@@ -127,6 +127,7 @@ func monitorDNSBL(log *mlog.Log) {
 	}
 }
 
+// also see localserve.go, code is similar or even shared.
 func cmdServe(c *cmd) {
 	c.help = `Start mox, serving SMTP/IMAP/HTTPS.
 
@@ -320,38 +321,6 @@ requested, other TLS certificates are requested on demand.
 
 	go monitorDNSBL(log)
 
-	shutdown := func() {
-		// We indicate we are shutting down. Causes new connections and new SMTP commands
-		// to be rejected. Should stop active connections pretty quickly.
-		mox.ShutdownCancel()
-
-		// Now we are going to wait for all connections to be gone, up to a timeout.
-		done := mox.Connections.Done()
-		second := time.Tick(time.Second)
-		select {
-		case <-done:
-			log.Print("connections shutdown, waiting until 1 second passed")
-			<-second
-
-		case <-time.Tick(3 * time.Second):
-			// We now cancel all pending operations, and set an immediate deadline on sockets.
-			// Should get us a clean shutdown relatively quickly.
-			mox.ContextCancel()
-			mox.Connections.Shutdown()
-
-			second := time.Tick(time.Second)
-			select {
-			case <-done:
-				log.Print("no more connections, shutdown is clean, waiting until 1 second passed")
-				<-second // Still wait for second, giving processes like imports a chance to clean up.
-			case <-second:
-				log.Print("shutting down with pending sockets")
-			}
-		}
-		err := os.Remove(mox.DataDirPath("ctl"))
-		log.Check(err, "removing ctl unix domain socket during shutdown")
-	}
-
 	ctlpath := mox.DataDirPath("ctl")
 	_ = os.Remove(ctlpath)
 	ctl, err := net.Listen("unix", ctlpath)
@@ -367,7 +336,7 @@ requested, other TLS certificates are requested on demand.
 			}
 			cid := mox.Cid()
 			ctx := context.WithValue(mox.Context, mlog.CidKey, cid)
-			go servectl(ctx, log.WithCid(cid), conn, shutdown)
+			go servectl(ctx, log.WithCid(cid), conn, func() { shutdown(log) })
 		}
 	}()
 
@@ -398,7 +367,7 @@ requested, other TLS certificates are requested on demand.
 	signal.Notify(sigc, os.Interrupt, syscall.SIGTERM)
 	sig := <-sigc
 	log.Print("shutting down, waiting max 3s for existing connections", mlog.Field("signal", sig))
-	shutdown()
+	shutdown(log)
 	if num, ok := sig.(syscall.Signal); ok {
 		os.Exit(int(num))
 	} else {
@@ -406,6 +375,38 @@ requested, other TLS certificates are requested on demand.
 	}
 }
 
+func shutdown(log *mlog.Log) {
+	// We indicate we are shutting down. Causes new connections and new SMTP commands
+	// to be rejected. Should stop active connections pretty quickly.
+	mox.ShutdownCancel()
+
+	// Now we are going to wait for all connections to be gone, up to a timeout.
+	done := mox.Connections.Done()
+	second := time.Tick(time.Second)
+	select {
+	case <-done:
+		log.Print("connections shutdown, waiting until 1 second passed")
+		<-second
+
+	case <-time.Tick(3 * time.Second):
+		// We now cancel all pending operations, and set an immediate deadline on sockets.
+		// Should get us a clean shutdown relatively quickly.
+		mox.ContextCancel()
+		mox.Connections.Shutdown()
+
+		second := time.Tick(time.Second)
+		select {
+		case <-done:
+			log.Print("no more connections, shutdown is clean, waiting until 1 second passed")
+			<-second // Still wait for second, giving processes like imports a chance to clean up.
+		case <-second:
+			log.Print("shutting down with pending sockets")
+		}
+	}
+	err := os.Remove(mox.DataDirPath("ctl"))
+	log.Check(err, "removing ctl unix domain socket during shutdown")
+}
+
 // Set correct permissions for mox working directory, binary, config and data and service file.
 //
 // We require being able to stat the basic non-optional paths. Then we'll try to

smtpserver/reputation_test.go

@@ -55,17 +55,7 @@ func TestReputation(t *testing.T) {
 			}
 		}
 
-		var ipmasked1, ipmasked2, ipmasked3 string
+		ipmasked1, ipmasked2, ipmasked3 := ipmasked(net.ParseIP(ip))
-		var xip = net.ParseIP(ip)
-		if xip.To4() != nil {
-			ipmasked1 = xip.String()
-			ipmasked2 = xip.Mask(net.CIDRMask(26, 32)).String()
-			ipmasked3 = xip.Mask(net.CIDRMask(21, 32)).String()
-		} else {
-			ipmasked1 = xip.Mask(net.CIDRMask(64, 128)).String()
-			ipmasked2 = xip.Mask(net.CIDRMask(48, 128)).String()
-			ipmasked3 = xip.Mask(net.CIDRMask(32, 128)).String()
-		}
 
 		uidgen++
 		m := store.Message{

smtpserver/server.go

@@ -19,6 +19,7 @@ import (
 	"net"
 	"os"
 	"runtime/debug"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -61,6 +62,10 @@ var xlog = mlog.New("smtpserver")
 // These errors signal the connection must be closed.
 var errIO = errors.New("fatal io error")
 
+// If set, regular delivery/submit is sidestepped, email is accepted and
+// delivered to the account named mox.
+var Localserve bool
+
 var limiterConnectionRate, limiterConnections *ratelimit.Limiter
 
 // For delivery rate limiting. Variable because changed during tests.
@@ -1287,6 +1292,11 @@ func (c *conn) cmdMail(p *parser) {
 		c.log.Info("delivery from address without domain", mlog.Field("mailfrom", rpath.String()))
 		xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SePol7Other0, "domain name required")
 	}
+
+	if Localserve && strings.HasPrefix(string(rpath.Localpart), "mailfrom") {
+		c.xlocalserveError(rpath.Localpart)
+	}
+
 	c.mailFrom = &rpath
 
 	c.bwritecodeline(smtp.C250Completed, smtp.SeAddr1Other0, "looking good", nil)
@@ -1347,7 +1357,7 @@ func (c *conn) cmdRcpt(p *parser) {
 	// ../rfc/5321:3598
 	// ../rfc/5321:4045
 	// Also see ../rfc/7489:2214
-	if !c.submission && len(c.recipients) == 1 {
+	if !c.submission && len(c.recipients) == 1 && !Localserve {
 		// note: because of check above, mailFrom cannot be the null address.
 		var pass bool
 		d := c.mailFrom.IPDomain.Domain
@@ -1376,7 +1386,18 @@ func (c *conn) cmdRcpt(p *parser) {
 		}
 	}
 
-	if len(fpath.IPDomain.IP) > 0 {
+	if Localserve {
+		if strings.HasPrefix(string(fpath.Localpart), "rcptto") {
+			c.xlocalserveError(fpath.Localpart)
+		}
+
+		// If account or destination doesn't exist, it will be handled during delivery. For
+		// submissions, which is the common case, we'll deliver to the logged in user,
+		// which is typically the mox user.
+		acc, _ := mox.Conf.Account("mox")
+		dest := acc.Destinations["mox@localhost"]
+		c.recipients = append(c.recipients, rcptAccount{fpath, true, "mox", dest, "mox@localhost"})
+	} else if len(fpath.IPDomain.IP) > 0 {
 		if !c.submission {
 			xsmtpUserErrorf(smtp.C550MailboxUnavail, smtp.SeAddr1UnknownDestMailbox1, "not accepting email for ip")
 		}
@@ -1674,23 +1695,95 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 	}
 	msgPrefix = append(msgPrefix, []byte(authResults.Header())...)
 
-	for i, rcptAcc := range c.recipients {
+	if Localserve {
-		xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...)
+		var timeout bool
-		// todo: don't convert the headers to a body? it seems the body part is optional. does this have consequences for us in other places? ../rfc/5322:343
+		c.account.WithWLock(func() {
-		if !msgWriter.HaveHeaders {
+			for i, rcptAcc := range c.recipients {
-			xmsgPrefix = append(xmsgPrefix, "\r\n"...)
+				var code int
+				code, timeout = localserveNeedsError(rcptAcc.rcptTo.Localpart)
+				if timeout {
+					// Get out of wlock, and sleep there.
+					return
+				} else if code != 0 {
+					c.log.Info("failure due to special localpart", mlog.Field("code", code))
+					xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code)
+				}
+
+				xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...)
+				// todo: don't convert the headers to a body? it seems the body part is optional. does this have consequences for us in other places? ../rfc/5322:343
+				if !msgWriter.HaveHeaders {
+					xmsgPrefix = append(xmsgPrefix, "\r\n"...)
+				}
+				msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
+
+				ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP)
+				m := store.Message{
+					Received:           time.Now(),
+					RemoteIP:           c.remoteIP.String(),
+					RemoteIPMasked1:    ipmasked1,
+					RemoteIPMasked2:    ipmasked2,
+					RemoteIPMasked3:    ipmasked3,
+					EHLODomain:         c.hello.Domain.Name(),
+					MailFrom:           c.mailFrom.String(),
+					MailFromLocalpart:  c.mailFrom.Localpart,
+					MailFromDomain:     c.mailFrom.IPDomain.Domain.Name(),
+					RcptToLocalpart:    rcptAcc.rcptTo.Localpart,
+					RcptToDomain:       rcptAcc.rcptTo.IPDomain.Domain.Name(),
+					MsgFromLocalpart:   msgFrom.Localpart,
+					MsgFromDomain:      msgFrom.Domain.Name(),
+					MsgFromOrgDomain:   publicsuffix.Lookup(ctx, msgFrom.Domain).Name(),
+					EHLOValidated:      true,
+					MailFromValidated:  true,
+					MsgFromValidated:   true,
+					EHLOValidation:     store.ValidationPass,
+					MailFromValidation: store.ValidationRelaxed,
+					MsgFromValidation:  store.ValidationRelaxed,
+					DKIMDomains:        nil,
+					Size:               msgSize,
+					MsgPrefix:          xmsgPrefix,
+				}
+
+				if err := c.account.Deliver(c.log, rcptAcc.destination, &m, dataFile, i == len(c.recipients)-1); err != nil {
+					// Aborting the transaction is not great. But continuing and generating DSNs will
+					// probably result in errors as well...
+					metricSubmission.WithLabelValues("localserveerror").Inc()
+					c.log.Errorx("delivering message", err)
+					xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err)
+				}
+				metricSubmission.WithLabelValues("ok").Inc()
+				c.log.Info("submitted message delivered", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize))
+			}
+		})
+
+		if timeout {
+			c.log.Info("timing out submission due to special localpart")
+			mox.Sleep(mox.Context, time.Hour)
+			xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out submission due to special localpart")
 		}
 
-		msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
+	} else {
-		if err := queue.Add(c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, xmsgPrefix, dataFile, nil, i == len(c.recipients)-1); err != nil {
+		// We always deliver through the queue. It would be more efficient to deliver
-			// Aborting the transaction is not great. But continuing and generating DSNs will
+		// directly, but we don't want to circumvent all the anti-spam measures. Accounts
-			// probably result in errors as well...
+		// on a single mox instance should be allowed to block each other.
-			metricSubmission.WithLabelValues("queueerror").Inc()
+
-			c.log.Errorx("queuing message", err)
+		for i, rcptAcc := range c.recipients {
-			xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err)
+			xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...)
+			// todo: don't convert the headers to a body? it seems the body part is optional. does this have consequences for us in other places? ../rfc/5322:343
+			if !msgWriter.HaveHeaders {
+				xmsgPrefix = append(xmsgPrefix, "\r\n"...)
+			}
+
+			msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
+			if err := queue.Add(c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, xmsgPrefix, dataFile, nil, i == len(c.recipients)-1); err != nil {
+				// Aborting the transaction is not great. But continuing and generating DSNs will
+				// probably result in errors as well...
+				metricSubmission.WithLabelValues("queueerror").Inc()
+				c.log.Errorx("queuing message", err)
+				xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err)
+			}
+			metricSubmission.WithLabelValues("ok").Inc()
+			c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize))
 		}
-		metricSubmission.WithLabelValues("ok").Inc()
-		c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize))
 	}
 	err = dataFile.Close()
 	c.log.Check(err, "closing file after submission")
@@ -1703,6 +1796,55 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 	c.writecodeline(smtp.C250Completed, smtp.SeMailbox2Other0, "it is done", nil)
 }
 
+func ipmasked(ip net.IP) (string, string, string) {
+	if ip.To4() != nil {
+		m1 := ip.String()
+		m2 := ip.Mask(net.CIDRMask(26, 32)).String()
+		m3 := ip.Mask(net.CIDRMask(21, 32)).String()
+		return m1, m2, m3
+	}
+	m1 := ip.Mask(net.CIDRMask(64, 128)).String()
+	m2 := ip.Mask(net.CIDRMask(48, 128)).String()
+	m3 := ip.Mask(net.CIDRMask(32, 128)).String()
+	return m1, m2, m3
+}
+
+func localserveNeedsError(lp smtp.Localpart) (code int, timeout bool) {
+	s := string(lp)
+	if strings.HasSuffix(s, "temperror") {
+		return smtp.C451LocalErr, false
+	} else if strings.HasSuffix(s, "permerror") {
+		return smtp.C550MailboxUnavail, false
+	} else if strings.HasSuffix(s, "timeout") {
+		return 0, true
+	}
+	if len(s) < 3 {
+		return 0, false
+	}
+	s = s[len(s)-3:]
+	v, err := strconv.ParseInt(s, 10, 32)
+	if err != nil {
+		return 0, false
+	}
+	if v < 400 || v > 600 {
+		return 0, false
+	}
+	return int(v), false
+}
+
+func (c *conn) xlocalserveError(lp smtp.Localpart) {
+	code, timeout := localserveNeedsError(lp)
+	if timeout {
+		c.log.Info("timing out due to special localpart")
+		mox.Sleep(mox.Context, time.Hour)
+		xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "timing out command due to special localpart")
+	} else if code != 0 {
+		c.log.Info("failure due to special localpart", mlog.Field("code", code))
+		metricDelivery.WithLabelValues("delivererror", "localserve").Inc()
+		xsmtpServerErrorf(codes{code, smtp.SeOther00}, "failure with code %d due to special localpart", code)
+	}
+}
+
 // deliver is called for incoming messages from external, typically untrusted
 // sources. i.e. not submitted by authenticated users.
 func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgWriter *message.Writer, iprevStatus iprev.Status, pdataFile **os.File) {
@@ -1968,16 +2110,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 	c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain))
 
 	// Prepare for analyzing content, calculating reputation.
-	var ipmasked1, ipmasked2, ipmasked3 string
+	ipmasked1, ipmasked2, ipmasked3 := ipmasked(c.remoteIP)
-	if c.remoteIP.To4() != nil {
-		ipmasked1 = c.remoteIP.String()
-		ipmasked2 = c.remoteIP.Mask(net.CIDRMask(26, 32)).String()
-		ipmasked3 = c.remoteIP.Mask(net.CIDRMask(21, 32)).String()
-	} else {
-		ipmasked1 = c.remoteIP.Mask(net.CIDRMask(64, 128)).String()
-		ipmasked2 = c.remoteIP.Mask(net.CIDRMask(48, 128)).String()
-		ipmasked3 = c.remoteIP.Mask(net.CIDRMask(32, 128)).String()
-	}
 	var verifiedDKIMDomains []string
 	for _, r := range dkimResults {
 		// A message can have multiple signatures for the same identity. For example when
@@ -2019,7 +2152,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 
 	// For each recipient, do final spam analysis and delivery.
 	for _, rcptAcc := range c.recipients {
-
 		log := c.log.Fields(mlog.Field("mailfrom", c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo))
 
 		// If this is not a valid local user, we send back a DSN. This can only happen when
@@ -2238,34 +2370,47 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 			mox.Sleep(mox.Context, reputationlessSenderDeliveryDelay)
 		}
 
-		acc.WithWLock(func() {
+		// Gather the message-id before we deliver and the file may be consumed.
-			// Gather the message-id before we deliver and the file may be consumed.
+		if !parsedMessageID {
-			if !parsedMessageID {
+			if p, err := message.Parse(store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil {
-				if p, err := message.Parse(store.FileMsgReader(m.MsgPrefix, dataFile)); err != nil {
+				log.Infox("parsing message for message-id", err)
-					log.Infox("parsing message for message-id", err)
+			} else if header, err := p.Header(); err != nil {
-				} else if header, err := p.Header(); err != nil {
+				log.Infox("parsing message header for message-id", err)
-					log.Infox("parsing message header for message-id", err)
+			} else {
-				} else {
+				messageID = header.Get("Message-Id")
-					messageID = header.Get("Message-Id")
-				}
 			}
+		}
 
-			if err := acc.Deliver(log, rcptAcc.destination, m, dataFile, false); err != nil {
+		if Localserve {
-				log.Errorx("delivering", err)
+			code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart)
-				metricDelivery.WithLabelValues("delivererror", a.reason).Inc()
+			if timeout {
-				addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing")
+				c.log.Info("timing out due to special localpart")
-				return
+				mox.Sleep(mox.Context, time.Hour)
+				xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeOther00}, "timing out delivery due to special localpart")
+			} else if code != 0 {
+				c.log.Info("failure due to special localpart", mlog.Field("code", code))
+				metricDelivery.WithLabelValues("delivererror", "localserve").Inc()
+				addError(rcptAcc, code, smtp.SeOther00, false, fmt.Sprintf("failure with code %d due to special localpart", code))
 			}
-			metricDelivery.WithLabelValues("delivered", a.reason).Inc()
+		} else {
-			log.Info("incoming message delivered", mlog.Field("reason", a.reason))
+			acc.WithWLock(func() {
-
+				if err := acc.Deliver(log, rcptAcc.destination, m, dataFile, false); err != nil {
-			conf, _ := acc.Conf()
+					log.Errorx("delivering", err)
-			if conf.RejectsMailbox != "" && messageID != "" {
+					metricDelivery.WithLabelValues("delivererror", a.reason).Inc()
-				if err := acc.RejectsRemove(log, conf.RejectsMailbox, messageID); err != nil {
+					addError(rcptAcc, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing")
-					log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID))
+					return
 				}
-			}
+				metricDelivery.WithLabelValues("delivered", a.reason).Inc()
-		})
+				log.Info("incoming message delivered", mlog.Field("reason", a.reason))
+
+				conf, _ := acc.Conf()
+				if conf.RejectsMailbox != "" && messageID != "" {
+					if err := acc.RejectsRemove(log, conf.RejectsMailbox, messageID); err != nil {
+						log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID))
+					}
+				}
+			})
+		}
 
 		err = acc.Close()
 		log.Check(err, "closing account after delivering")
@@ -2347,7 +2492,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 		}
 		dsnMsg.Original = header
 
-		if err := queueDSN(c, *c.mailFrom, dsnMsg); err != nil {
+		if Localserve {
+			c.log.Error("not queueing dsn for incoming delivery due to localserve")
+		} else if err := queueDSN(c, *c.mailFrom, dsnMsg); err != nil {
 			metricServerErrors.WithLabelValues("queuedsn").Inc()
 			c.log.Errorx("queuing DSN for incoming delivery, no DSN sent", err)
 		}