From 36470f9d955ae99e60ddc36dfa519c12d5a358bb Mon Sep 17 00:00:00 2001 From: Harald Rudell Date: Sat, 27 Jan 2024 08:18:57 -0800 Subject: [PATCH] imaptemp.NewImapTemp runs temporary-storage mox imap-server listening on returned bidirectional stream for integration testing --- imapserver/internals.go | 23 +++ imaptemp/close-oncer.go | 44 +++++ imaptemp/domains.conf | 21 +++ imaptemp/imap-temp.go | 363 +++++++++++++++++++++++++++++++++++++ imaptemp/imap-temp_test.go | 74 ++++++++ imaptemp/mox.conf | 15 ++ imaptemp/no-deadline.go | 28 +++ mlog/log.go | 13 ++ 8 files changed, 581 insertions(+) create mode 100644 imapserver/internals.go create mode 100644 imaptemp/close-oncer.go create mode 100644 imaptemp/domains.conf create mode 100644 imaptemp/imap-temp.go create mode 100644 imaptemp/imap-temp_test.go create mode 100644 imaptemp/mox.conf create mode 100644 imaptemp/no-deadline.go diff --git a/imapserver/internals.go b/imapserver/internals.go new file mode 100644 index 0000000..db56fd5 --- /dev/null +++ b/imapserver/internals.go @@ -0,0 +1,23 @@ +package imapserver + +import ( + "crypto/tls" + "net" +) + +// internalsStruct exports some private top-level identifiers +type internalsStruct struct { + // LimitersInit configures mox’ connection rate limiting and max connections + LimitersInit func() + // Serve launches the mox imap server with storage using provided connection + Serve func(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, xtls bool, noRequireSTARTTLS bool) +} + +// internalsValue exports some private top-level identifiers +var internalsValue = internalsStruct{ + LimitersInit: limitersInit, + Serve: serve, +} + +// Internals exports some private top-level identifiers +func Internals() (internals internalsStruct) { return internalsValue } diff --git a/imaptemp/close-oncer.go b/imaptemp/close-oncer.go new file mode 100644 index 0000000..8979ae5 --- /dev/null +++ b/imaptemp/close-oncer.go @@ -0,0 +1,44 @@ +/* +Written 2024 by Harald Rudell (https://haraldrudell.github.io/haraldrudell/) +*/ + +package imaptemp + +import ( + "net" + "sync/atomic" +) + +// closeOncer ensures a net.Conn to have thread-safe idempotent observable Close +type closeOncer struct { + net.Conn + CloseWait chan struct{} + IsClosed atomic.Bool +} + +// newCloseOncer returns a wrapper making the net.Conn to have thread-safe idempotent observable Close +func newCloseOncer(conn net.Conn) (c net.Conn) { + return &closeOncer{ + Conn: conn, + CloseWait: make(chan struct{}), + } +} + +// Close is thread-safe idempotent observable +// - only the winner thread actually closing receives a possible error +// - no thread returns prior to the connection being closed +// - Close can be awaited by <-c.CloseWait +// - Close is observable by c.IsClosed.Load() +func (c *closeOncer) Close() (err error) { + if !c.IsClosed.CompareAndSwap(false, true) { + // loser thread gets to wait + <-c.CloseWait + return + } + defer close(c.CloseWait) + + // winner thread does close + err = c.Conn.Close() + + return +} diff --git a/imaptemp/domains.conf b/imaptemp/domains.conf new file mode 100644 index 0000000..35c9800 --- /dev/null +++ b/imaptemp/domains.conf @@ -0,0 +1,21 @@ +Domains: + mox.example: + LocalpartCaseSensitive: false +Accounts: + mjl: + Domain: mox.example + Destinations: + mjl@mox.example: nil + ""@mox.example: nil + JunkFilter: + Threshold: 0.95 + Params: + Twograms: true + MaxPower: 0.1 + TopWords: 10 + IgnoreWords: 0.1 + limit: + Domain: mox.example + Destinations: + limit@mox.example: nil + QuotaMessageSize: 1 diff --git a/imaptemp/imap-temp.go b/imaptemp/imap-temp.go new file mode 100644 index 0000000..8419160 --- /dev/null +++ b/imaptemp/imap-temp.go @@ -0,0 +1,363 @@ +/* +Written 2024 by Harald Rudell (https://haraldrudell.github.io/haraldrudell/) +*/ + +package imaptemp + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "sync/atomic" + + "golang.org/x/exp/slog" + + _ "embed" + + "github.com/mjl-/mox/imapserver" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/store" +) + +// NewImapMem returns an object that runs an temporary-storage imap server +// - configurable: logging, timeouts, file-system location +// - logger is optional logger. If missing or nil, logging is to standard error, text with all log-levels +// - — [golang.org/x/exp/slog.New] creates a logger that may control: +// - — what [io.Writer] to send logging to, eg. standard output or file +// - — log format, eg. text or json +// - — log level eg. [github.com/mjl-/mox/mlog.LevelError] +// - because package [github.com/mjl-/mox/store] is not thread-safe: +// - — only one instance at any one time per process +// - — If regular mox code is used, that and CreateSocket can only be invoked by a single thread due to lack of thread-safety +// - file-system storage depends on process’ working directory +// - — multiple processes must be in different working directories +// +// Example: +// +// var options = slog.HandlerOptions{ +// Level: minLevel, +// } +// levelFilteredLogger = slog.New(slog.NewTextHandler(os.Stdout, &options)) +func NewImapTemp(logger ...*slog.Logger) (imapTemp *ImapTemp) { + var _ slog.TextHandler + + var createNewStructuredLogger *slog.Logger + if len(logger) > 0 { + createNewStructuredLogger = logger[0] + if createNewStructuredLogger != nil { + mlog.LogModel.Store(createNewStructuredLogger) + } + } + return &ImapTemp{ + pkglog: mlog.New(logPackageLabel, createNewStructuredLogger), + } +} + +// UseTimeouts ensures that mox’ socket timeouts are active and cause disconnections +// - as default, mox timeouts aree disabled allowing to debug and single step for +// extended periods of time +func (i *ImapTemp) UseTimeouts() { i.useTimeouts.Store(true) } + +// CreateSocket returns a socket where an IMAP4Rev2 server is listening +// - imapConn.Close is thread-safe idempotent +// - if CreateSocket returns successfully, Close must be invoked to release resources +// - if dir is provided, it is a file-system directory where mox wil keep its data +// - CreateSocket can only be invoked once per ImapTemp instance +// - only one ImapTemp can be active at a time per process +// - for thread-safety, regular mox code should not be used in a process where CreateSocket is used +// - thread-safe +// - mox start/stop takes around 500 ms +// - — +// - the returned socket imapConn is an emulated network socket bypassing networking instead connecting +// directly to the server via an unbuffered Go channel: +// - — no TLS +// - — no TCP port +// - — [io.ReadWriteCloser] over unbuffered channels +// - a server thread is launched +func (i *ImapTemp) CreateSocket(dir ...string) (imapConn net.Conn, err error) { + if len(dir) > 0 { + if i.dir, err = filepath.Abs(dir[0]); err != nil { + return + } + } + + // ensure that CreateSocket is only invoked once for this ImapTemp instance + if !i.isCreateServer.CompareAndSwap(false, true) { + err = errors.New("CreateSocket invoked more than once") + return + } + + // to enforce one mox at a time per process, + // the gating value is a private top-level variable + // - the consumer must assure that: + // - — regular mox is not used in a non-thread-safe way at any time in a process using ImapTemp + if !isRunning.CompareAndSwap(nil, i) { + err = errors.New("ImapTemp is already running") + return + } + defer i.createEnd(&err) + // this ImapTemp now owns mox until isRunning is set to nil + + // create a unique file-system directory for storage + if i.dir == "" { + const useDefaultTempDir = "" + if i.dir, err = os.MkdirTemp(useDefaultTempDir, dirTemplate); err != nil { + return + } + i.noRemove.Store(true) + } + + // make mox able to run + internals.LimitersInit() + // store.InitialUIDValidity is not thread-safe + // - reading and writing isRunning makes access thread-safe for ImapTemp instances + i.initialUIDValidity = store.InitialUIDValidity + store.InitialUIDValidity = validityOne + // here we must copy mox test code into the real world + // - server_test.go:337 startArgs + // server new connection rate-limits + // mox handles cancel of this context + mox.Context = context.Background() + // write “mox.conf” + mox.ConfigStaticPath = filepath.Join(i.dir, moxConf) + if err = os.WriteFile(mox.ConfigStaticPath, moxConfData, urwx); err != nil { + return + } + // write “domains.conf” + mox.ConfigDynamicPath = filepath.Join(i.dir, domainsConf) + if err = os.WriteFile(mox.ConfigDynamicPath, domainsConfData, urwx); err != nil { + return + } + // do not load certificates from configuration + const doNotLoadTLSKeyCerts = false + // do not load certificate renewal data from configuration + const doNotCheckACMEHosts = false + // errs is a list of non-nil errors + if errs := mox.LoadConfig(context.Background(), i.pkglog, doNotLoadTLSKeyCerts, doNotCheckACMEHosts); len(errs) > 0 { + err = errs[0] + return + } + // create account: opening it will create + var acc *store.Account + if acc, err = store.OpenAccount(i.pkglog, accountName); err != nil { + return + // the account will have a password + } else if err = acc.SetPassword(i.pkglog, passwd); err != nil { + return + } + + // launch server thread + + // some notification bus + var switchStopFunc = store.Switchboard() + // the socket used by the server + // - imapConn is the connection where IMAP4rev2 protocol is available + var connUsedByServer net.Conn + connUsedByServer, imapConn = net.Pipe() + // disable all deadlines on connUsedByServer + if !i.useTimeouts.Load() { + connUsedByServer = NewNoDeadline(connUsedByServer) + } + // make imapConn.Close idempotent + // - an imap client may close on: + // - — server closing or + // - — protocol error or + // - — after sending imap LOGOUT + // - making Close idempotent allows for the connection to with certainty be closed exactly once + imapConn = newCloseOncer(imapConn) + i.imapConn = imapConn + // make server-thread awaitable + var serverExit = make(chan struct{}) + i.serverExit = serverExit + go i.imapServerThread(connUsedByServer, switchStopFunc, serverExit) + + return +} + +// After successful CreateSocket, EndCh is available +// - EndCh returns a channel that closes once resources are released +// - thread-safe +func (i *ImapTemp) EndCh() (endCh <-chan struct{}) { + i.isCreateServer.Load() + endCh = i.serverExit + return +} + +// Creds returns login credentials for the imap server’s only account +// - thread-safe +func (i *ImapTemp) Creds() (emailAcccountName, password string) { return accountEmail, passwd } + +// Dir returns absolute path to temporary file-system storage +// - no consumer should need this +// - thread-safe +func (i *ImapTemp) Dir() (dir string) { + i.isCreateServer.Load() + dir = i.dir + return +} + +// Close releases resources from a successful CreateSocket invocation +// - thread-safe +func (i *ImapTemp) Close() (err error) { + if isRunning.Load() != i { + return // this is not an active imapTemp + } + err = i.unCreate() + + return +} + +// imapServerThread runs a server until ? +func (i *ImapTemp) imapServerThread(connUsedByServer net.Conn, switchStopFunc func(), serverExit chan<- struct{}) { + defer close(serverExit) + defer i.recoverPanic(recover()) + defer switchStopFunc() + + // do not use TLS requiring certificates and such + const isTLS = false + // do not use TLS requiring certificates and such + const allowLoginWithoutTLS = true + var noTLSConfig *tls.Config + // cid is numbering of connection used for logging + var cid = connectionNumberer.Add(1) + internals.Serve(listenerName, cid, noTLSConfig, connUsedByServer, isTLS, allowLoginWithoutTLS) +} + +// recoverPanic stores any server-thread runtime error at ImapTemp.serverPanic +func (i *ImapTemp) recoverPanic(panicValue any) { + var err error + var ok bool + if panicValue == nil { + return + } else if err, ok = panicValue.(error); !ok { + err = fmt.Errorf("server-thread PANIC: non-error value: %T %+[1]v", panicValue) + } else { + err = fmt.Errorf("server-thread PANIC: %w", err) + } + i.serverPanic = err +} + +// createEnd invokes unCreate if CreateSocket failed +func (i *ImapTemp) createEnd(errp *error) { + if *errp == nil { + i.isCreateServer.Store(true) // make any writes thread-safe + return // CreateSocket success return + } + // an error is already present, so ignore error here + _ = i.unCreate() +} + +// unCreate undos any actions by CreateSocket +// - only inoked on: +// - — CreateSocket failing +// - — Close after CreateSocket success +func (i *ImapTemp) unCreate() (err error) { + i.isCreateServer.Load() // make any previous writes thread-safe + defer isRunning.Store(nil) // release mox for other instances + + store.InitialUIDValidity = i.initialUIDValidity + if closer := i.imapConn; closer != nil { + err = closer.Close() + } + if serverExit := i.serverExit; serverExit != nil { + <-serverExit + if e := i.serverPanic; e != nil { + if err == nil { + err = e + } else { + i.pkglog.Error(e.Error()) + } + } + } + // remove temporary files, ignore error if an error is already present + if !i.noRemove.Load() { + if e := os.RemoveAll(i.dir); e != nil && err == nil { + err = e + } + } + + return +} + +//go:embed mox.conf +var moxConfData []byte + +//go:embed domains.conf +var domainsConfData []byte + +// validityOne returns UIDValidity 1 during ImapTemp being active +func validityOne() (UIDValidity uint32) { return 1 } + +const ( + // naming template for temporary file-system directory + dirTemplate = "mox" + // configuration file is hard-coded in the test code + moxConf = "mox.conf" + // some filename mox needs + domainsConf = "domains.conf" + // log label is hard-coded in test code + logPackageLabel = "imapserver" + // accountName is imap server account login name + // - returned by Creds method + accountName = "mjl" + // impa LOGIN must be by email + accountEmail = accountName + "@mox.example" + // imap server account password + // - returned by Creds method + passwd = "testtest" + // listenerName is a label the server outputs when the connection is ready + listenerName = "test" + // filemode for created files + urwx os.FileMode = 0700 +) + +// retrieve private identifiers from imapserver package +var internals = imapserver.Internals() + +// connectionNumberer number connections 1… for logging +var connectionNumberer atomic.Int64 + +// isRunning ensures that only one ImapMem runs at any one time per process +// - because mox uses non-thread-safe top-level variables, it is a process-scope top-level variable +// - provides thread-safety for [store.InitialUIDValidity] and possibly the rest of mox +var isRunning atomic.Pointer[ImapTemp] + +// ImapTemp runs an IMAP4Rev2 server with temporary storage for testing purposes +type ImapTemp struct { + // pkglog is a standard error logger labeled “imapserver” + pkglog mlog.Log + // gates CreateSocket invocations + // - provides field thread-safety + isCreateServer atomic.Bool + // true if timeouts should not be disabled + useTimeouts atomic.Bool + // true if file system is not temporary and should not be removed + noRemove atomic.Bool + + // absolute path to temporary file-system storage + // - thread-safe through isCreateServer read and write + dir string + // held temporary value + // - thread-safe through isCreateServer read and write + initialUIDValidity func() uint32 + // imap connection to close on unCreate + // - may be nil + // - thread-safe through isCreateServer read and write + imapConn io.Closer + // serverExit closes on server thread exit + // - after successful Create value is available. + // - this channel will close on imap server thread-exit + // - may be nil + // - thread-safe through isCreateServer read and write + serverExit <-chan struct{} + // possible server panic + // - available after non-nil serverExit closes + // - thread-safe through isCreateServer read and write + serverPanic error +} diff --git a/imaptemp/imap-temp_test.go b/imaptemp/imap-temp_test.go new file mode 100644 index 0000000..d082cf2 --- /dev/null +++ b/imaptemp/imap-temp_test.go @@ -0,0 +1,74 @@ +/* +Written 2024 by Harald Rudell (https://haraldrudell.github.io/haraldrudell/) +*/ + +package imaptemp + +import ( + "bufio" + "bytes" + "net" + "testing" +) + +func TestImapTemp(t *testing.T) { + //t.Errorf("logging on") + + // expected prefix of the greeting “* OK” + var prefixExp = "* OK " + // expected suffix of the greeting “mox imap\r\n” + var suffixExp = "mox imap\r\n" + // the greeting ends with the newline byte + const newLine = '\n' + + var imapTemp *ImapTemp + var err error + var imapConn net.Conn + var lineReader *bufio.Reader + var line []byte + + imapTemp = NewImapTemp() + + imapConn, err = imapTemp.CreateSocket() + defer closeImapTemp(imapTemp, t) + + // CreateSocket should succeed + if err != nil { + t.Errorf("CreateSocket err: %q", err) + return + } + + // IMAP4rev2 server should output a greeting + // - read until first newline, unlimited length, no timeout + lineReader = bufio.NewReader(imapConn) + if line, err = lineReader.ReadBytes(newLine); err != nil { + t.Errorf("ReadBytes err: %q", err) + return + } + + // line: "* OK [CAPABILITY + // IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ + // IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES + // MOVE UTF8=ACCEPT LIST-EXTENDED SPECIAL-USE LIST-STATUS + // AUTH=SCRAM-SHA-256-PLUS AUTH=SCRAM-SHA-256 + // AUTH=SCRAM-SHA-1-PLUS AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 + // ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC + // STATUS=SIZE AUTH=PLAIN + // ] mox imap\r\n" + t.Logf("line: %q", line) + + // greeting should contain OK and mox + if !bytes.HasPrefix(line, []byte(prefixExp)) || !bytes.HasSuffix(line, []byte(suffixExp)) { + t.Errorf("FAIL bad greeting from mox: prefixExp: %q suffixExp %q greeting:\n%s", + prefixExp, suffixExp, line, + ) + } +} + +// closeImapTemp is a deferrable function closing imapTemp +func closeImapTemp(imapTemp *ImapTemp, t *testing.T) { + var err = imapTemp.Close() + if err != nil { + t.Errorf("FAIL imapTemp.Close err: %q", err) + } +} diff --git a/imaptemp/mox.conf b/imaptemp/mox.conf new file mode 100644 index 0000000..3ed95b9 --- /dev/null +++ b/imaptemp/mox.conf @@ -0,0 +1,15 @@ +DataDir: data +User: 1000 +LogLevel: trace +Hostname: mox.example +Listeners: + local: + IPs: + - 0.0.0.0 + IMAP: + Enabled: true + Port: 1143 + NoRequireSTARTTLS: true +Postmaster: + Account: mjl + Mailbox: postmaster diff --git a/imaptemp/no-deadline.go b/imaptemp/no-deadline.go new file mode 100644 index 0000000..51367b2 --- /dev/null +++ b/imaptemp/no-deadline.go @@ -0,0 +1,28 @@ +/* +Written 2024 by Harald Rudell (https://haraldrudell.github.io/haraldrudell/) +*/ + +package imaptemp + +import ( + "net" + "time" +) + +// NoDeadline is a wrapper for a net.Conn socket that disables attempts to set a timeout on the socket +type NoDeadline struct{ net.Conn } + +// NewNoDeadline wraps conn disabling any attempt to set a timeout on the socket +func NewNoDeadline(conn net.Conn) (noDeadline *NoDeadline) { return &NoDeadline{Conn: conn} } + +// SetDeadline sets the read and write deadlines associated +// with the connection. +func (n *NoDeadline) SetDeadline(t time.Time) (err error) { return } + +// SetReadDeadline sets the deadline for future Read calls +// and any currently-blocked Read call. +func (n *NoDeadline) SetReadDeadline(t time.Time) (err error) { return } + +// SetWriteDeadline sets the deadline for future Write calls +// and any currently-blocked Write call. +func (n *NoDeadline) SetWriteDeadline(t time.Time) (err error) { return } diff --git a/mlog/log.go b/mlog/log.go index 6619ef4..6dcfb55 100644 --- a/mlog/log.go +++ b/mlog/log.go @@ -28,6 +28,16 @@ import ( "golang.org/x/exp/slog" ) +// LogModel allows to control new loggers created by mox packages +// - must be set prior to instantiating mox objects +// - default is text logging to standard output of all levels +// - mox has custom log levels like [LevelInfo] +// - value: [golang.org/x/exp/slog.New] creates a logger that may control: +// - — what [io.Writer] to send logging to, eg. standard output [os.Stdout] or file +// - — log format, eg. text or json +// - — log level eg. [github.com/mjl-/mox/mlog.LevelError] +var LogModel atomic.Pointer[slog.Logger] + var noctx = context.Background() // Logfmt enabled output in logfmt, instead of output more suitable for @@ -111,6 +121,9 @@ type Log struct { // New returns a Log that adds a "pkg" attribute. If logger is nil, a new // Logger is created with a custom handler. func New(pkg string, logger *slog.Logger) Log { + if logger == nil { + logger = LogModel.Load() + } if logger == nil { logger = slog.New(&handler{}) }