serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-27 08:53:48 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

queue: implement adding a message to the queue that gets sent to multiple recipients

Browse source 
and in a way that allows us to send that message to multiple recipients in a
single smtp transaction.
This commit is contained in:
Mechiel Lukkien 2024-03-05 20:10:28 +01:00
parent 15e450df61
commit 47ebfa8152
No known key found for this signature in database
15 changed files with 346 additions and 276 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
dmarcdb/eval.go
View file

@ -842,13 +842,13 @@ Period: %s - %s UTC
continue continue
} }
qm := queue.MakeMsg(mox.Conf.Static.Postmaster.Account, from.Path(), rcpt.address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil) qm := queue.MakeMsg(from.Path(), rcpt.address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil)
// Don't try as long as regular deliveries, and stop before we would send the // Don't try as long as regular deliveries, and stop before we would send the
// delayed DSN. Though we also won't send that due to IsDMARCReport. // delayed DSN. Though we also won't send that due to IsDMARCReport.
qm.MaxAttempts = 5 qm.MaxAttempts = 5
qm.IsDMARCReport = true qm.IsDMARCReport = true
err = queueAdd(ctx, log, &qm, msgf) err = queueAdd(ctx, log, mox.Conf.Static.Postmaster.Account, msgf, qm)
if err != nil { if err != nil {
tempError = true tempError = true
log.Errorx("queueing message with dmarc aggregate report", err) log.Errorx("queueing message with dmarc aggregate report", err)
@ -997,13 +997,13 @@ Submitting-URI: %s
continue continue
} }
qm := queue.MakeMsg(mox.Conf.Static.Postmaster.Account, fromAddr.Path(), rcpt.Address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil) qm := queue.MakeMsg(fromAddr.Path(), rcpt.Address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil)
// Don't try as long as regular deliveries, and stop before we would send the // Don't try as long as regular deliveries, and stop before we would send the
// delayed DSN. Though we also won't send that due to IsDMARCReport. // delayed DSN. Though we also won't send that due to IsDMARCReport.
qm.MaxAttempts = 5 qm.MaxAttempts = 5
qm.IsDMARCReport = true qm.IsDMARCReport = true
if err := queueAdd(ctx, log, &qm, msgf); err != nil { if err := queueAdd(ctx, log, mox.Conf.Static.Postmaster.Account, msgf, qm); err != nil {
log.Errorx("queueing message with dmarc error report", err) log.Errorx("queueing message with dmarc error report", err)
metricReportError.Inc() metricReportError.Inc()
} else { } else {

7
dmarcdb/eval_test.go
View file

@ -295,7 +295,12 @@ func TestSendReports(t *testing.T) {
aggrAddrs := map[string]struct{}{} aggrAddrs := map[string]struct{}{}
errorAddrs := map[string]struct{}{} errorAddrs := map[string]struct{}{}
queueAdd = func(ctx context.Context, log mlog.Log, qm *queue.Msg, msgFile *os.File) error { queueAdd = func(ctx context.Context, log mlog.Log, senderAccount string, msgFile *os.File, qml ...queue.Msg) error {
if len(qml) != 1 {
return fmt.Errorf("queued %d messages, expected 1", len(qml))
}
qm := qml[0]
// Read message file. Also write copy to disk for inspection. // Read message file. Also write copy to disk for inspection.
buf, err := io.ReadAll(&moxio.AtReader{R: msgFile}) buf, err := io.ReadAll(&moxio.AtReader{R: msgFile})
tcheckf(t, err, "read report message") tcheckf(t, err, "read report message")

4
gentestdata.go
View file

@ -233,8 +233,8 @@ Accounts:
const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n" const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
_, err = fmt.Fprint(mf, qmsg) _, err = fmt.Fprint(mf, qmsg)
xcheckf(err, "writing message") xcheckf(err, "writing message")
qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil) qm := queue.MakeMsg(mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
err = queue.Add(ctxbg, c.log, &qm, mf) err = queue.Add(ctxbg, c.log, "test0", mf, qm)
xcheckf(err, "enqueue message") xcheckf(err, "enqueue message")
// Create three accounts. // Create three accounts.

80
queue/queue.go
View file

@ -79,6 +79,13 @@ var Localserve bool
// queueing related fields. // queueing related fields.
type Msg struct { type Msg struct {
ID int64 ID int64
// A message for multiple recipients will get a BaseID that is identical to the
// first Msg.ID queued. They may be delivered in a single SMTP transaction if they
// are going to the same mail server. For messages with a single recipient, this
// field will be 0.
BaseID int64 `bstore:"index"`
Queued time.Time `bstore:"default now"` Queued time.Time `bstore:"default now"`
SenderAccount string // Failures are delivered back to this local account. Also used for routing. SenderAccount string // Failures are delivered back to this local account. Also used for routing.
SenderLocalpart smtp.Localpart // Should be a local user and domain. SenderLocalpart smtp.Localpart // Should be a local user and domain.
@ -208,10 +215,9 @@ func Count(ctx context.Context) (int, error) {
} }
// MakeMsg is a convenience function that sets the commonly used fields for a Msg. // MakeMsg is a convenience function that sets the commonly used fields for a Msg.
func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, prefix []byte, requireTLS *bool) Msg { func MakeMsg(sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, prefix []byte, requireTLS *bool) Msg {
now := time.Now() now := time.Now()
return Msg{ return Msg{
SenderAccount: senderAccount,
SenderLocalpart: sender.Localpart, SenderLocalpart: sender.Localpart,
SenderDomain: sender.IPDomain, SenderDomain: sender.IPDomain,
RecipientLocalpart: recipient.Localpart, RecipientLocalpart: recipient.Localpart,
@ -228,25 +234,31 @@ func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf
} }
} }
// Add a new message to the queue. The queue is kicked immediately to start a // Add one or more new messages to the queue. They'll get the same BaseID, so they
// first delivery attempt. // can be delivered in a single SMTP transaction, with a single DATA command, but
// may be split into multiple transactions if errors/limits are encountered. The
// queue is kicked immediately to start a first delivery attempt.
// //
// ID must be 0 and will be set after inserting in the queue. // ID of the messagse must be 0 and will be set after inserting in the queue.
// //
// Add sets derived fields like RecipientDomainStr, and fields related to queueing, // Add sets derived fields like RecipientDomainStr, and fields related to queueing,
// such as Queued, NextAttempt, LastAttempt, LastError. // such as Queued, NextAttempt, LastAttempt, LastError.
func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error { func Add(ctx context.Context, log mlog.Log, senderAccount string, msgFile *os.File, qml ...Msg) error {
// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759 if len(qml) == 0 {
return fmt.Errorf("must queue at least one message")
}
for _, qm := range qml {
if qm.ID != 0 { if qm.ID != 0 {
return fmt.Errorf("id of queued message must be 0") return fmt.Errorf("id of queued messages must be 0")
}
} }
if Localserve { if Localserve {
if qm.SenderAccount == "" { if senderAccount == "" {
return fmt.Errorf("cannot queue with localserve without local account") return fmt.Errorf("cannot queue with localserve without local account")
} }
acc, err := store.OpenAccount(log, qm.SenderAccount) acc, err := store.OpenAccount(log, senderAccount)
if err != nil { if err != nil {
return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err) return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
} }
@ -254,17 +266,24 @@ func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error {
err := acc.Close() err := acc.Close()
log.Check(err, "closing account") log.Check(err, "closing account")
}() }()
m := store.Message{Size: qm.Size, MsgPrefix: qm.MsgPrefix}
conf, _ := acc.Conf() conf, _ := acc.Conf()
dest := conf.Destinations[qm.Sender().String()] err = nil
acc.WithWLock(func() { acc.WithWLock(func() {
for i, qm := range qml {
qml[i].SenderAccount = senderAccount
m := store.Message{Size: qm.Size, MsgPrefix: qm.MsgPrefix}
dest := conf.Destinations[qm.Sender().String()]
err = acc.DeliverDestination(log, dest, &m, msgFile) err = acc.DeliverDestination(log, dest, &m, msgFile)
})
if err != nil { if err != nil {
return fmt.Errorf("delivering message: %v", err) err = fmt.Errorf("delivering message: %v", err)
return // Returned again outside WithWLock.
} }
}
})
if err == nil {
log.Debug("immediately delivered from queue to sender") log.Debug("immediately delivered from queue to sender")
return nil }
return err
} }
tx, err := DB.Begin(ctx, true) tx, err := DB.Begin(ctx, true)
@ -279,17 +298,35 @@ func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error {
} }
}() }()
if err := tx.Insert(qm); err != nil { // Insert messages into queue. If there are multiple messages, they all get a
// non-zero BaseID that is the Msg.ID of the first message inserted.
var baseID int64
for i := range qml {
qml[i].SenderAccount = senderAccount
qml[i].BaseID = baseID
if err := tx.Insert(&qml[i]); err != nil {
return err return err
} }
if i == 0 && len(qml) > 1 {
baseID = qml[i].ID
qml[i].BaseID = baseID
if err := tx.Update(&qml[i]); err != nil {
return err
}
}
}
dst := qm.MessagePath() var paths []string
defer func() { defer func() {
if dst != "" { for _, p := range paths {
err := os.Remove(dst) err := os.Remove(p)
log.Check(err, "removing destination message file for queue", slog.String("path", dst)) log.Check(err, "removing destination message file for queue", slog.String("path", p))
} }
}() }()
for _, qm := range qml {
dst := qm.MessagePath()
paths = append(paths, dst)
dstDir := filepath.Dir(dst) dstDir := filepath.Dir(dst)
os.MkdirAll(dstDir, 0770) os.MkdirAll(dstDir, 0770)
if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil { if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
@ -297,12 +334,13 @@ func Add(ctx context.Context, log mlog.Log, qm *Msg, msgFile *os.File) error {
} else if err := moxio.SyncDir(log, dstDir); err != nil { } else if err := moxio.SyncDir(log, dstDir); err != nil {
return fmt.Errorf("sync directory: %v", err) return fmt.Errorf("sync directory: %v", err)
} }
}
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
return fmt.Errorf("commit transaction: %s", err) return fmt.Errorf("commit transaction: %s", err)
} }
tx = nil tx = nil
dst = "" paths = nil
queuekick() queuekick()
return nil return nil

92
queue/queue_test.go
View file

@ -117,12 +117,12 @@ func TestQueue(t *testing.T) {
var qm Msg var qm Msg
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qm)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qm)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg) msgs, err = List(ctxbg)
@ -451,8 +451,8 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of its route. // Add a message to be delivered with submit because of its route.
topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}} topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}}
qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg(path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qm)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
wasNetDialer = testDeliver(fakeSubmitServer) wasNetDialer = testDeliver(fakeSubmitServer)
if !wasNetDialer { if !wasNetDialer {
@ -460,11 +460,11 @@ func TestQueue(t *testing.T) {
} }
// Add a message to be delivered with submit because of explicitly configured transport, that uses TLS. // Add a message to be delivered with submit because of explicitly configured transport, that uses TLS.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qml := []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
transportSubmitTLS := "submittls" transportSubmitTLS := "submittls"
n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS) n, err = Kick(ctxbg, qml[0].ID, "", "", &transportSubmitTLS)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -509,11 +509,11 @@ func TestQueue(t *testing.T) {
} }
// Add a message to be delivered with socks. // Add a message to be delivered with socks.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
transportSocks := "socks" transportSocks := "socks"
n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks) n, err = Kick(ctxbg, qml[0].ID, "", "", &transportSocks)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -525,10 +525,10 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with opportunistic TLS verification. // Add message to be delivered with opportunistic TLS verification.
clearTLSResults(t) clearTLSResults(t)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -539,10 +539,10 @@ func TestQueue(t *testing.T) {
// Test fallback to plain text with TLS handshake fails. // Test fallback to plain text with TLS handshake fails.
clearTLSResults(t) clearTLSResults(t)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -559,10 +559,10 @@ func TestQueue(t *testing.T) {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo}, {Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo},
}, },
} }
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -580,10 +580,10 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with verified TLS and REQUIRETLS. // Add message to be delivered with verified TLS and REQUIRETLS.
yes := true yes := true
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -597,10 +597,10 @@ func TestQueue(t *testing.T) {
{}, {},
}, },
} }
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -618,10 +618,10 @@ func TestQueue(t *testing.T) {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)}, {Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)},
}, },
} }
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -640,10 +640,10 @@ func TestQueue(t *testing.T) {
// Check that message is delivered with TLS-Required: No and non-matching DANE record. // Check that message is delivered with TLS-Required: No and non-matching DANE record.
no := false no := false
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -651,10 +651,10 @@ func TestQueue(t *testing.T) {
testDeliver(fakeSMTPSTARTTLSServer) testDeliver(fakeSMTPSTARTTLSServer)
// Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text. // Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -662,10 +662,10 @@ func TestQueue(t *testing.T) {
testDeliver(makeBadFakeSMTPSTARTTLSServer(true)) testDeliver(makeBadFakeSMTPSTARTTLSServer(true))
// Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers. // Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -677,10 +677,10 @@ func TestQueue(t *testing.T) {
resolver.TLSA = nil resolver.TLSA = nil
// Add message with requiretls that fails immediately due to no verification policy for recipient domain. // Add message with requiretls that fails immediately due to no verification policy for recipient domain.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes) qml = []Msg{MakeMsg(path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes)}
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qml...)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, qm.ID, "", "", nil) n, err = Kick(ctxbg, qml[0].ID, "", "", nil)
tcheck(t, err, "kick queue") tcheck(t, err, "kick queue")
if n != 1 { if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n) t.Fatalf("kick changed %d messages, expected 1", n)
@ -692,8 +692,8 @@ func TestQueue(t *testing.T) {
}) })
// Add another message that we'll fail to deliver entirely. // Add another message that we'll fail to deliver entirely.
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm = MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qm)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg) msgs, err = List(ctxbg)
@ -883,8 +883,8 @@ func TestQueueStart(t *testing.T) {
mf := prepareFile(t) mf := prepareFile(t)
defer os.Remove(mf.Name()) defer os.Remove(mf.Name())
defer mf.Close() defer mf.Close()
qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil) qm := MakeMsg(path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, pkglog, &qm, mf) err = Add(ctxbg, pkglog, "mjl", mf, qm)
tcheck(t, err, "add message to queue for delivery") tcheck(t, err, "add message to queue for delivery")
checkDialed(true) checkDialed(true)

1
rfc/index.txt
View file

@ -97,6 +97,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
8601 Yes - Message Header Field for Indicating Message Authentication Status 8601 Yes - Message Header Field for Indicating Message Authentication Status
8689 Yes - SMTP Require TLS Option 8689 Yes - SMTP Require TLS Option
8904 No - DNS Whitelist (DNSWL) Email Authentication Method Extension 8904 No - DNS Whitelist (DNSWL) Email Authentication Method Extension
9422 Partial - The LIMITS SMTP Service Extension
# SPF # SPF
4408 Yes Obs (by RFC 7208) Sender Policy Framework (SPF) for Authorizing Use of Domains in E-Mail, Version 1 4408 Yes Obs (by RFC 7208) Sender Policy Framework (SPF) for Authorizing Use of Domains in E-Mail, Version 1

4
smtpserver/dsn.go
View file

@ -53,9 +53,9 @@ func queueDSN(ctx context.Context, log mlog.Log, c *conn, rcptTo smtp.Path, m ds
if requireTLS { if requireTLS {
reqTLS = &requireTLS reqTLS = &requireTLS
} }
qm := queue.MakeMsg("", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, reqTLS) qm := queue.MakeMsg(smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, reqTLS)
qm.DSNUTF8 = bufUTF8 qm.DSNUTF8 = bufUTF8
if err := queue.Add(ctx, c.log, &qm, f); err != nil { if err := queue.Add(ctx, c.log, "", f, qm); err != nil {
return err return err
} }
return nil return nil

39
smtpserver/server.go
View file

@ -72,6 +72,10 @@ var limiterConnectionRate, limiterConnections *ratelimit.Limiter
var limitIPMasked1MessagesPerMinute int = 500 var limitIPMasked1MessagesPerMinute int = 500
var limitIPMasked1SizePerMinute int64 = 1000 * 1024 * 1024 var limitIPMasked1SizePerMinute int64 = 1000 * 1024 * 1024
// Maximum number of RCPT TO commands (i.e. recipients) for a single message
// delivery. Must be at least 100. Announced in LIMIT extension.
const rcptToLimit = 1000
func init() { func init() {
// Also called by tests, so they don't trigger the rate limiter. // Also called by tests, so they don't trigger the rate limiter.
limitersInit() limitersInit()
@ -889,6 +893,7 @@ func (c *conn) cmdHello(p *parser, ehlo bool) {
c.bwritelinef("250-ENHANCEDSTATUSCODES") // ../rfc/2034:71 c.bwritelinef("250-ENHANCEDSTATUSCODES") // ../rfc/2034:71
// todo future? c.writelinef("250-DSN") // todo future? c.writelinef("250-DSN")
c.bwritelinef("250-8BITMIME") // ../rfc/6152:86 c.bwritelinef("250-8BITMIME") // ../rfc/6152:86
c.bwritelinef("250-LIMITS RCPTMAX=%d", rcptToLimit) // rfc/9422:301
c.bwritecodeline(250, "", "SMTPUTF8", nil) // ../rfc/6531:201 c.bwritecodeline(250, "", "SMTPUTF8", nil) // ../rfc/6531:201
c.xflush() c.xflush()
} }
@ -1556,9 +1561,9 @@ func (c *conn) cmdRcpt(p *parser) {
// todo future: for submission, should we do explicit verification that domains are fully qualified? also for mail from. ../rfc/6409:420 // todo future: for submission, should we do explicit verification that domains are fully qualified? also for mail from. ../rfc/6409:420
if len(c.recipients) >= 100 { if len(c.recipients) >= rcptToLimit {
// ../rfc/5321:3535 ../rfc/5321:3571 // ../rfc/5321:3535 ../rfc/5321:3571
xsmtpUserErrorf(smtp.C452StorageFull, smtp.SeProto5TooManyRcpts3, "max of 100 recipients reached") xsmtpUserErrorf(smtp.C452StorageFull, smtp.SeProto5TooManyRcpts3, "max of %d recipients reached", rcptToLimit)
} }
// We don't want to allow delivery to multiple recipients with a null reverse path. // We don't want to allow delivery to multiple recipients with a null reverse path.
@ -1974,7 +1979,8 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
// We always deliver through the queue. It would be more efficient to deliver // We always deliver through the queue. It would be more efficient to deliver
// directly, but we don't want to circumvent all the anti-spam measures. Accounts // directly, but we don't want to circumvent all the anti-spam measures. Accounts
// on a single mox instance should be allowed to block each other. // on a single mox instance should be allowed to block each other.
for _, rcptAcc := range c.recipients { qml := make([]queue.Msg, len(c.recipients))
for i, rcptAcc := range c.recipients {
if Localserve { if Localserve {
code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart) code, timeout := localserveNeedsError(rcptAcc.rcptTo.Localpart)
if timeout { if timeout {
@ -1988,15 +1994,17 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
} }
xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...) xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...)
msgSize := int64(len(xmsgPrefix)) + msgWriter.Size msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
qm := queue.MakeMsg(c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS) qm := queue.MakeMsg(*c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS)
if !c.futureRelease.IsZero() { if !c.futureRelease.IsZero() {
qm.NextAttempt = c.futureRelease qm.NextAttempt = c.futureRelease
qm.FutureReleaseRequest = c.futureReleaseRequest qm.FutureReleaseRequest = c.futureReleaseRequest
} }
qml[i] = qm
}
// todo: it would be good to have a limit on messages (count and total size) a user has in the queue. also/especially with futurerelease. ../rfc/4865:387 // todo: it would be good to have a limit on messages (count and total size) a user has in the queue. also/especially with futurerelease. ../rfc/4865:387
if err := queue.Add(ctx, c.log, &qm, dataFile); err != nil { if err := queue.Add(ctx, c.log, c.account.Name, dataFile, qml...); err != nil {
// Aborting the transaction is not great. But continuing and generating DSNs will // Aborting the transaction is not great. But continuing and generating DSNs will
// probably result in errors as well... // probably result in errors as well...
metricSubmission.WithLabelValues("queueerror").Inc() metricSubmission.WithLabelValues("queueerror").Inc()
@ -2004,16 +2012,25 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err) xsmtpServerErrorf(errCodes(smtp.C451LocalErr, smtp.SeSys3Other0, err), "error delivering message: %v", err)
} }
metricSubmission.WithLabelValues("ok").Inc() metricSubmission.WithLabelValues("ok").Inc()
c.log.Info("message queued for delivery", for i, rcptAcc := range c.recipients {
c.log.Info("messages queued for delivery",
slog.Any("mailfrom", *c.mailFrom), slog.Any("mailfrom", *c.mailFrom),
slog.Any("rcptto", rcptAcc.rcptTo), slog.Any("rcptto", rcptAcc.rcptTo),
slog.Bool("smtputf8", c.smtputf8), slog.Bool("smtputf8", c.smtputf8),
slog.Int64("msgsize", msgSize)) slog.Int64("msgsize", qml[i].Size))
err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
xcheckf(err, "adding outgoing message")
} }
err = c.account.DB.Write(ctx, func(tx *bstore.Tx) error {
for _, rcptAcc := range c.recipients {
outgoing := store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)}
if err := tx.Insert(&outgoing); err != nil {
return fmt.Errorf("adding outgoing message: %v", err)
}
}
return nil
})
xcheckf(err, "adding outgoing messages")
c.transactionGood++ c.transactionGood++
c.transactionBad-- // Compensate for early earlier pessimistic increase. c.transactionBad-- // Compensate for early earlier pessimistic increase.

4
tlsrptsend/send.go
View file

@ -589,7 +589,7 @@ Period: %s - %s UTC
continue continue
} }
qm := queue.MakeMsg(mox.Conf.Static.Postmaster.Account, from.Path(), rcpt.Address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil) qm := queue.MakeMsg(from.Path(), rcpt.Address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil)
// Don't try as long as regular deliveries, and stop before we would send the // Don't try as long as regular deliveries, and stop before we would send the
// delayed DSN. Though we also won't send that due to IsTLSReport. // delayed DSN. Though we also won't send that due to IsTLSReport.
// ../rfc/8460:1077 // ../rfc/8460:1077
@ -599,7 +599,7 @@ Period: %s - %s UTC
no := false no := false
qm.RequireTLS = &no qm.RequireTLS = &no
err = queueAdd(ctx, log, &qm, msgf) err = queueAdd(ctx, log, mox.Conf.Static.Postmaster.Account, msgf, qm)
if err != nil { if err != nil {
tempError = !queued tempError = !queued
log.Errorx("queueing message with tls report", err) log.Errorx("queueing message with tls report", err)

10
tlsrptsend/send_test.go
View file

@ -412,7 +412,11 @@ func TestSendReports(t *testing.T) {
var mutex sync.Mutex var mutex sync.Mutex
var index int var index int
queueAdd = func(ctx context.Context, log mlog.Log, qm *queue.Msg, msgFile *os.File) error { queueAdd = func(ctx context.Context, log mlog.Log, senderAccount string, msgFile *os.File, qml ...queue.Msg) error {
if len(qml) != 1 {
return fmt.Errorf("queued %d messages, expect 1", len(qml))
}
mutex.Lock() mutex.Lock()
defer mutex.Unlock() defer mutex.Unlock()
@ -421,13 +425,13 @@ func TestSendReports(t *testing.T) {
tcheckf(t, err, "read report message") tcheckf(t, err, "read report message")
p := fmt.Sprintf("../testdata/tlsrptsend/data/report%d.eml", index) p := fmt.Sprintf("../testdata/tlsrptsend/data/report%d.eml", index)
index++ index++
err = os.WriteFile(p, append(append([]byte{}, qm.MsgPrefix...), buf...), 0600) err = os.WriteFile(p, append(append([]byte{}, qml[0].MsgPrefix...), buf...), 0600)
tcheckf(t, err, "write report message") tcheckf(t, err, "write report message")
reportJSON, err := tlsrpt.ParseMessage(log.Logger, msgFile) reportJSON, err := tlsrpt.ParseMessage(log.Logger, msgFile)
tcheckf(t, err, "parsing generated report message") tcheckf(t, err, "parsing generated report message")
addr := qm.Recipient().String() addr := qml[0].Recipient().String()
haveReports[addr] = append(haveReports[addr], reportJSON.Convert()) haveReports[addr] = append(haveReports[addr], reportJSON.Convert())
return nil return nil

12
webadmin/admin.js
View file

@ -393,7 +393,7 @@ var api;
"Reverse": { "Name": "Reverse", "Docs": "", "Fields": [{ "Name": "Hostnames", "Docs": "", "Typewords": ["[]", "string"] }] }, "Reverse": { "Name": "Reverse", "Docs": "", "Fields": [{ "Name": "Hostnames", "Docs": "", "Typewords": ["[]", "string"] }] },
"ClientConfigs": { "Name": "ClientConfigs", "Docs": "", "Fields": [{ "Name": "Entries", "Docs": "", "Typewords": ["[]", "ClientConfigsEntry"] }] }, "ClientConfigs": { "Name": "ClientConfigs", "Docs": "", "Fields": [{ "Name": "Entries", "Docs": "", "Typewords": ["[]", "ClientConfigsEntry"] }] },
"ClientConfigsEntry": { "Name": "ClientConfigsEntry", "Docs": "", "Fields": [{ "Name": "Protocol", "Docs": "", "Typewords": ["string"] }, { "Name": "Host", "Docs": "", "Typewords": ["Domain"] }, { "Name": "Port", "Docs": "", "Typewords": ["int32"] }, { "Name": "Listener", "Docs": "", "Typewords": ["string"] }, { "Name": "Note", "Docs": "", "Typewords": ["string"] }] }, "ClientConfigsEntry": { "Name": "ClientConfigsEntry", "Docs": "", "Fields": [{ "Name": "Protocol", "Docs": "", "Typewords": ["string"] }, { "Name": "Host", "Docs": "", "Typewords": ["Domain"] }, { "Name": "Port", "Docs": "", "Typewords": ["int32"] }, { "Name": "Listener", "Docs": "", "Typewords": ["string"] }, { "Name": "Note", "Docs": "", "Typewords": ["string"] }] },
"Msg": { "Name": "Msg", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Queued", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SenderAccount", "Docs": "", "Typewords": ["string"] }, { "Name": "SenderLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "SenderDomain", "Docs": "", "Typewords": ["IPDomain"] }, { "Name": "RecipientLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RecipientDomain", "Docs": "", "Typewords": ["IPDomain"] }, { "Name": "RecipientDomainStr", "Docs": "", "Typewords": ["string"] }, { "Name": "Attempts", "Docs": "", "Typewords": ["int32"] }, { "Name": "MaxAttempts", "Docs": "", "Typewords": ["int32"] }, { "Name": "DialedIPs", "Docs": "", "Typewords": ["{}", "[]", "IP"] }, { "Name": "NextAttempt", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "LastAttempt", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "LastError", "Docs": "", "Typewords": ["string"] }, { "Name": "Has8bit", "Docs": "", "Typewords": ["bool"] }, { "Name": "SMTPUTF8", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsDMARCReport", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsTLSReport", "Docs": "", "Typewords": ["bool"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "DSNUTF8", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Transport", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureReleaseRequest", "Docs": "", "Typewords": ["string"] }] }, "Msg": { "Name": "Msg", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "BaseID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Queued", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "SenderAccount", "Docs": "", "Typewords": ["string"] }, { "Name": "SenderLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "SenderDomain", "Docs": "", "Typewords": ["IPDomain"] }, { "Name": "RecipientLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RecipientDomain", "Docs": "", "Typewords": ["IPDomain"] }, { "Name": "RecipientDomainStr", "Docs": "", "Typewords": ["string"] }, { "Name": "Attempts", "Docs": "", "Typewords": ["int32"] }, { "Name": "MaxAttempts", "Docs": "", "Typewords": ["int32"] }, { "Name": "DialedIPs", "Docs": "", "Typewords": ["{}", "[]", "IP"] }, { "Name": "NextAttempt", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "LastAttempt", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "LastError", "Docs": "", "Typewords": ["string"] }, { "Name": "Has8bit", "Docs": "", "Typewords": ["bool"] }, { "Name": "SMTPUTF8", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsDMARCReport", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsTLSReport", "Docs": "", "Typewords": ["bool"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "DSNUTF8", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Transport", "Docs": "", "Typewords": ["string"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "FutureReleaseRequest", "Docs": "", "Typewords": ["string"] }] },
"IPDomain": { "Name": "IPDomain", "Docs": "", "Fields": [{ "Name": "IP", "Docs": "", "Typewords": ["IP"] }, { "Name": "Domain", "Docs": "", "Typewords": ["Domain"] }] }, "IPDomain": { "Name": "IPDomain", "Docs": "", "Fields": [{ "Name": "IP", "Docs": "", "Typewords": ["IP"] }, { "Name": "Domain", "Docs": "", "Typewords": ["Domain"] }] },
"WebserverConfig": { "Name": "WebserverConfig", "Docs": "", "Fields": [{ "Name": "WebDNSDomainRedirects", "Docs": "", "Typewords": ["[]", "[]", "Domain"] }, { "Name": "WebDomainRedirects", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "WebHandlers", "Docs": "", "Typewords": ["[]", "WebHandler"] }] }, "WebserverConfig": { "Name": "WebserverConfig", "Docs": "", "Fields": [{ "Name": "WebDNSDomainRedirects", "Docs": "", "Typewords": ["[]", "[]", "Domain"] }, { "Name": "WebDomainRedirects", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "WebHandlers", "Docs": "", "Typewords": ["[]", "WebHandler"] }] },
"WebHandler": { "Name": "WebHandler", "Docs": "", "Fields": [{ "Name": "LogName", "Docs": "", "Typewords": ["string"] }, { "Name": "Domain", "Docs": "", "Typewords": ["string"] }, { "Name": "PathRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "DontRedirectPlainHTTP", "Docs": "", "Typewords": ["bool"] }, { "Name": "Compress", "Docs": "", "Typewords": ["bool"] }, { "Name": "WebStatic", "Docs": "", "Typewords": ["nullable", "WebStatic"] }, { "Name": "WebRedirect", "Docs": "", "Typewords": ["nullable", "WebRedirect"] }, { "Name": "WebForward", "Docs": "", "Typewords": ["nullable", "WebForward"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "DNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, "WebHandler": { "Name": "WebHandler", "Docs": "", "Fields": [{ "Name": "LogName", "Docs": "", "Typewords": ["string"] }, { "Name": "Domain", "Docs": "", "Typewords": ["string"] }, { "Name": "PathRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "DontRedirectPlainHTTP", "Docs": "", "Typewords": ["bool"] }, { "Name": "Compress", "Docs": "", "Typewords": ["bool"] }, { "Name": "WebStatic", "Docs": "", "Typewords": ["nullable", "WebStatic"] }, { "Name": "WebRedirect", "Docs": "", "Typewords": ["nullable", "WebRedirect"] }, { "Name": "WebForward", "Docs": "", "Typewords": ["nullable", "WebForward"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "DNSDomain", "Docs": "", "Typewords": ["Domain"] }] },
@ -2692,14 +2692,13 @@ const queueList = async () => {
client.Transports(), client.Transports(),
]); ]);
const nowSecs = new Date().getTime() / 1000; const nowSecs = new Date().getTime() / 1000;
dom._kids(page, crumbs(crumblink('Mox Admin', '#'), 'Queue'), (msgs || []).length === 0 ? 'Currently no messages in the queue.' : [ dom._kids(page, crumbs(crumblink('Mox Admin', '#'), 'Queue'),
dom.p('The messages below are currently in the queue.'),
// todo: sorting by address/timestamps/attempts. perhaps filtering. // todo: sorting by address/timestamps/attempts. perhaps filtering.
dom.table(dom._class('hover'), dom.thead(dom.tr(dom.th('ID'), dom.th('Submitted'), dom.th('From'), dom.th('To'), dom.th('Size'), dom.th('Attempts'), dom.th('Next attempt'), dom.th('Last attempt'), dom.th('Last error'), dom.th('Require TLS'), dom.th('Transport/Retry'), dom.th('Remove'))), dom.tbody((msgs || []).map(m => { dom.table(dom._class('hover'), dom.thead(dom.tr(dom.th('ID'), dom.th('Submitted'), dom.th('From'), dom.th('To'), dom.th('Size'), dom.th('Attempts'), dom.th('Next attempt'), dom.th('Last attempt'), dom.th('Last error'), dom.th('Require TLS'), dom.th('Transport/Retry'), dom.th('Remove'))), dom.tbody((msgs || []).length === 0 ? dom.tr(dom.td(attr.colspan('12'), 'Currently no messages in the queue.')) : [], (msgs || []).map(m => {
let requiretlsFieldset; let requiretlsFieldset;
let requiretls; let requiretls;
let transport; let transport;
return dom.tr(dom.td('' + m.ID), dom.td(age(new Date(m.Queued), false, nowSecs)), dom.td(m.SenderLocalpart + "@" + ipdomainString(m.SenderDomain)), // todo: escaping of localpart return dom.tr(dom.td('' + m.ID + (m.BaseID > 0 ? '/' + m.BaseID : '')), dom.td(age(new Date(m.Queued), false, nowSecs)), dom.td(m.SenderLocalpart + "@" + ipdomainString(m.SenderDomain)), // todo: escaping of localpart
dom.td(m.RecipientLocalpart + "@" + ipdomainString(m.RecipientDomain)), // todo: escaping of localpart dom.td(m.RecipientLocalpart + "@" + ipdomainString(m.RecipientDomain)), // todo: escaping of localpart
dom.td(formatSize(m.Size)), dom.td('' + m.Attempts), dom.td(age(new Date(m.NextAttempt), true, nowSecs)), dom.td(m.LastAttempt ? age(new Date(m.LastAttempt), false, nowSecs) : '-'), dom.td(m.LastError || '-'), dom.td(dom.form(requiretlsFieldset = dom.fieldset(requiretls = dom.select(attr.title('How to use TLS for message delivery over SMTP:\n\nDefault: Delivery attempts follow the policies published by the recipient domain: Verification with MTA-STS and/or DANE, or optional opportunistic unverified STARTTLS if the domain does not specify a policy.\n\nWith RequireTLS: For sensitive messages, you may want to require verified TLS. The recipient destination domain SMTP server must support the REQUIRETLS SMTP extension for delivery to succeed. It is automatically chosen when the destination domain mail servers of all recipients are known to support it.\n\nFallback to insecure: If delivery fails due to MTA-STS and/or DANE policies specified by the recipient domain, and the content is not sensitive, you may choose to ignore the recipient domain TLS policies so delivery can succeed.'), dom.option('Default', attr.value('')), dom.option('With RequireTLS', attr.value('yes'), m.RequireTLS === true ? attr.selected('') : []), dom.option('Fallback to insecure', attr.value('no'), m.RequireTLS === false ? attr.selected('') : [])), ' ', dom.submitbutton('Save')), async function submit(e) { dom.td(formatSize(m.Size)), dom.td('' + m.Attempts), dom.td(age(new Date(m.NextAttempt), true, nowSecs)), dom.td(m.LastAttempt ? age(new Date(m.LastAttempt), false, nowSecs) : '-'), dom.td(m.LastError || '-'), dom.td(dom.form(requiretlsFieldset = dom.fieldset(requiretls = dom.select(attr.title('How to use TLS for message delivery over SMTP:\n\nDefault: Delivery attempts follow the policies published by the recipient domain: Verification with MTA-STS and/or DANE, or optional opportunistic unverified STARTTLS if the domain does not specify a policy.\n\nWith RequireTLS: For sensitive messages, you may want to require verified TLS. The recipient destination domain SMTP server must support the REQUIRETLS SMTP extension for delivery to succeed. It is automatically chosen when the destination domain mail servers of all recipients are known to support it.\n\nFallback to insecure: If delivery fails due to MTA-STS and/or DANE policies specified by the recipient domain, and the content is not sensitive, you may choose to ignore the recipient domain TLS policies so delivery can succeed.'), dom.option('Default', attr.value('')), dom.option('With RequireTLS', attr.value('yes'), m.RequireTLS === true ? attr.selected('') : []), dom.option('Fallback to insecure', attr.value('no'), m.RequireTLS === false ? attr.selected('') : [])), ' ', dom.submitbutton('Save')), async function submit(e) {
e.preventDefault(); e.preventDefault();
@ -2751,8 +2750,7 @@ const queueList = async () => {
} }
window.location.reload(); // todo: only refresh the list window.location.reload(); // todo: only refresh the list
}))); })));
}))), }))));
]);
}; };
const webserver = async () => { const webserver = async () => {
let conf = await client.WebserverConfig(); let conf = await client.WebserverConfig();

6
webadmin/admin.ts
View file

@ -2300,8 +2300,6 @@ const queueList = async () => {
crumblink('Mox Admin', '#'), crumblink('Mox Admin', '#'),
'Queue', 'Queue',
), ),
(msgs || []).length === 0 ? 'Currently no messages in the queue.' : [
dom.p('The messages below are currently in the queue.'),
// todo: sorting by address/timestamps/attempts. perhaps filtering. // todo: sorting by address/timestamps/attempts. perhaps filtering.
dom.table(dom._class('hover'), dom.table(dom._class('hover'),
dom.thead( dom.thead(
@ -2321,12 +2319,13 @@ const queueList = async () => {
), ),
), ),
dom.tbody( dom.tbody(
(msgs || []).length === 0 ? dom.tr(dom.td(attr.colspan('12'), 'Currently no messages in the queue.')) : [],
(msgs || []).map(m => { (msgs || []).map(m => {
let requiretlsFieldset: HTMLFieldSetElement let requiretlsFieldset: HTMLFieldSetElement
let requiretls: HTMLSelectElement let requiretls: HTMLSelectElement
let transport: HTMLSelectElement let transport: HTMLSelectElement
return dom.tr( return dom.tr(
dom.td(''+m.ID), dom.td(''+m.ID + (m.BaseID > 0 ? '/'+m.BaseID : '')),
dom.td(age(new Date(m.Queued), false, nowSecs)), dom.td(age(new Date(m.Queued), false, nowSecs)),
dom.td(m.SenderLocalpart+"@"+ipdomainString(m.SenderDomain)), // todo: escaping of localpart dom.td(m.SenderLocalpart+"@"+ipdomainString(m.SenderDomain)), // todo: escaping of localpart
dom.td(m.RecipientLocalpart+"@"+ipdomainString(m.RecipientDomain)), // todo: escaping of localpart dom.td(m.RecipientLocalpart+"@"+ipdomainString(m.RecipientDomain)), // todo: escaping of localpart
@ -2412,7 +2411,6 @@ const queueList = async () => {
}) })
), ),
), ),
],
) )
} }

7
webadmin/api.json
View file

@ -3370,6 +3370,13 @@
"int64" "int64"
] ]
}, },
{
"Name": "BaseID",
"Docs": "A message for multiple recipients will get a BaseID that is identical to the first Msg.ID queued. They may be delivered in a single SMTP transaction if they are going to the same mail server. For messages with a single recipient, this field will be 0.",
"Typewords": [
"int64"
]
},
{ {
"Name": "Queued", "Name": "Queued",
"Docs": "", "Docs": "",

3
webadmin/api.ts
View file

@ -471,6 +471,7 @@ export interface ClientConfigsEntry {
// queueing related fields. // queueing related fields.
export interface Msg { export interface Msg {
ID: number ID: number
BaseID: number // A message for multiple recipients will get a BaseID that is identical to the first Msg.ID queued. They may be delivered in a single SMTP transaction if they are going to the same mail server. For messages with a single recipient, this field will be 0.
Queued: Date Queued: Date
SenderAccount: string // Failures are delivered back to this local account. Also used for routing. SenderAccount: string // Failures are delivered back to this local account. Also used for routing.
SenderLocalpart: Localpart // Should be a local user and domain. SenderLocalpart: Localpart // Should be a local user and domain.
@ -837,7 +838,7 @@ export const types: TypenameMap = {
"Reverse": {"Name":"Reverse","Docs":"","Fields":[{"Name":"Hostnames","Docs":"","Typewords":["[]","string"]}]}, "Reverse": {"Name":"Reverse","Docs":"","Fields":[{"Name":"Hostnames","Docs":"","Typewords":["[]","string"]}]},
"ClientConfigs": {"Name":"ClientConfigs","Docs":"","Fields":[{"Name":"Entries","Docs":"","Typewords":["[]","ClientConfigsEntry"]}]}, "ClientConfigs": {"Name":"ClientConfigs","Docs":"","Fields":[{"Name":"Entries","Docs":"","Typewords":["[]","ClientConfigsEntry"]}]},
"ClientConfigsEntry": {"Name":"ClientConfigsEntry","Docs":"","Fields":[{"Name":"Protocol","Docs":"","Typewords":["string"]},{"Name":"Host","Docs":"","Typewords":["Domain"]},{"Name":"Port","Docs":"","Typewords":["int32"]},{"Name":"Listener","Docs":"","Typewords":["string"]},{"Name":"Note","Docs":"","Typewords":["string"]}]}, "ClientConfigsEntry": {"Name":"ClientConfigsEntry","Docs":"","Fields":[{"Name":"Protocol","Docs":"","Typewords":["string"]},{"Name":"Host","Docs":"","Typewords":["Domain"]},{"Name":"Port","Docs":"","Typewords":["int32"]},{"Name":"Listener","Docs":"","Typewords":["string"]},{"Name":"Note","Docs":"","Typewords":["string"]}]},
"Msg": {"Name":"Msg","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"Queued","Docs":"","Typewords":["timestamp"]},{"Name":"SenderAccount","Docs":"","Typewords":["string"]},{"Name":"SenderLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"SenderDomain","Docs":"","Typewords":["IPDomain"]},{"Name":"RecipientLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RecipientDomain","Docs":"","Typewords":["IPDomain"]},{"Name":"RecipientDomainStr","Docs":"","Typewords":["string"]},{"Name":"Attempts","Docs":"","Typewords":["int32"]},{"Name":"MaxAttempts","Docs":"","Typewords":["int32"]},{"Name":"DialedIPs","Docs":"","Typewords":["{}","[]","IP"]},{"Name":"NextAttempt","Docs":"","Typewords":["timestamp"]},{"Name":"LastAttempt","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"LastError","Docs":"","Typewords":["string"]},{"Name":"Has8bit","Docs":"","Typewords":["bool"]},{"Name":"SMTPUTF8","Docs":"","Typewords":["bool"]},{"Name":"IsDMARCReport","Docs":"","Typewords":["bool"]},{"Name":"IsTLSReport","Docs":"","Typewords":["bool"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"DSNUTF8","Docs":"","Typewords":["nullable","string"]},{"Name":"Transport","Docs":"","Typewords":["string"]},{"Name":"RequireTLS","Docs":"","Typewords":["nullable","bool"]},{"Name":"FutureReleaseRequest","Docs":"","Typewords":["string"]}]}, "Msg": {"Name":"Msg","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"BaseID","Docs":"","Typewords":["int64"]},{"Name":"Queued","Docs":"","Typewords":["timestamp"]},{"Name":"SenderAccount","Docs":"","Typewords":["string"]},{"Name":"SenderLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"SenderDomain","Docs":"","Typewords":["IPDomain"]},{"Name":"RecipientLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RecipientDomain","Docs":"","Typewords":["IPDomain"]},{"Name":"RecipientDomainStr","Docs":"","Typewords":["string"]},{"Name":"Attempts","Docs":"","Typewords":["int32"]},{"Name":"MaxAttempts","Docs":"","Typewords":["int32"]},{"Name":"DialedIPs","Docs":"","Typewords":["{}","[]","IP"]},{"Name":"NextAttempt","Docs":"","Typewords":["timestamp"]},{"Name":"LastAttempt","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"LastError","Docs":"","Typewords":["string"]},{"Name":"Has8bit","Docs":"","Typewords":["bool"]},{"Name":"SMTPUTF8","Docs":"","Typewords":["bool"]},{"Name":"IsDMARCReport","Docs":"","Typewords":["bool"]},{"Name":"IsTLSReport","Docs":"","Typewords":["bool"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"DSNUTF8","Docs":"","Typewords":["nullable","string"]},{"Name":"Transport","Docs":"","Typewords":["string"]},{"Name":"RequireTLS","Docs":"","Typewords":["nullable","bool"]},{"Name":"FutureReleaseRequest","Docs":"","Typewords":["string"]}]},
"IPDomain": {"Name":"IPDomain","Docs":"","Fields":[{"Name":"IP","Docs":"","Typewords":["IP"]},{"Name":"Domain","Docs":"","Typewords":["Domain"]}]}, "IPDomain": {"Name":"IPDomain","Docs":"","Fields":[{"Name":"IP","Docs":"","Typewords":["IP"]},{"Name":"Domain","Docs":"","Typewords":["Domain"]}]},
"WebserverConfig": {"Name":"WebserverConfig","Docs":"","Fields":[{"Name":"WebDNSDomainRedirects","Docs":"","Typewords":["[]","[]","Domain"]},{"Name":"WebDomainRedirects","Docs":"","Typewords":["[]","[]","string"]},{"Name":"WebHandlers","Docs":"","Typewords":["[]","WebHandler"]}]}, "WebserverConfig": {"Name":"WebserverConfig","Docs":"","Fields":[{"Name":"WebDNSDomainRedirects","Docs":"","Typewords":["[]","[]","Domain"]},{"Name":"WebDomainRedirects","Docs":"","Typewords":["[]","[]","string"]},{"Name":"WebHandlers","Docs":"","Typewords":["[]","WebHandler"]}]},
"WebHandler": {"Name":"WebHandler","Docs":"","Fields":[{"Name":"LogName","Docs":"","Typewords":["string"]},{"Name":"Domain","Docs":"","Typewords":["string"]},{"Name":"PathRegexp","Docs":"","Typewords":["string"]},{"Name":"DontRedirectPlainHTTP","Docs":"","Typewords":["bool"]},{"Name":"Compress","Docs":"","Typewords":["bool"]},{"Name":"WebStatic","Docs":"","Typewords":["nullable","WebStatic"]},{"Name":"WebRedirect","Docs":"","Typewords":["nullable","WebRedirect"]},{"Name":"WebForward","Docs":"","Typewords":["nullable","WebForward"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"DNSDomain","Docs":"","Typewords":["Domain"]}]}, "WebHandler": {"Name":"WebHandler","Docs":"","Fields":[{"Name":"LogName","Docs":"","Typewords":["string"]},{"Name":"Domain","Docs":"","Typewords":["string"]},{"Name":"PathRegexp","Docs":"","Typewords":["string"]},{"Name":"DontRedirectPlainHTTP","Docs":"","Typewords":["bool"]},{"Name":"Compress","Docs":"","Typewords":["bool"]},{"Name":"WebStatic","Docs":"","Typewords":["nullable","WebStatic"]},{"Name":"WebRedirect","Docs":"","Typewords":["nullable","WebRedirect"]},{"Name":"WebForward","Docs":"","Typewords":["nullable","WebForward"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"DNSDomain","Docs":"","Typewords":["Domain"]}]},

13
webmail/api.go
View file

@ -628,14 +628,15 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
Localpart: fromAddr.Address.Localpart, Localpart: fromAddr.Address.Localpart,
IPDomain: dns.IPDomain{Domain: fromAddr.Address.Domain}, IPDomain: dns.IPDomain{Domain: fromAddr.Address.Domain},
} }
for _, rcpt := range recipients { qml := make([]queue.Msg, len(recipients))
for i, rcpt := range recipients {
rcptMsgPrefix := recvHdrFor(rcpt.Pack(smtputf8)) + msgPrefix rcptMsgPrefix := recvHdrFor(rcpt.Pack(smtputf8)) + msgPrefix
msgSize := int64(len(rcptMsgPrefix)) + xc.Size msgSize := int64(len(rcptMsgPrefix)) + xc.Size
toPath := smtp.Path{ toPath := smtp.Path{
Localpart: rcpt.Localpart, Localpart: rcpt.Localpart,
IPDomain: dns.IPDomain{Domain: rcpt.Domain}, IPDomain: dns.IPDomain{Domain: rcpt.Domain},
} }
qm := queue.MakeMsg(reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS) qm := queue.MakeMsg(fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS)
if m.FutureRelease != nil { if m.FutureRelease != nil {
ival := time.Until(*m.FutureRelease) ival := time.Until(*m.FutureRelease)
if ival < 0 { if ival < 0 {
@ -647,13 +648,13 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
qm.FutureReleaseRequest = "until;" + m.FutureRelease.Format(time.RFC3339) qm.FutureReleaseRequest = "until;" + m.FutureRelease.Format(time.RFC3339)
// todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery. // todo: possibly add a header to the message stored in the Sent mailbox to indicate it was scheduled for later delivery.
} }
err := queue.Add(ctx, log, &qm, dataFile) qml[i] = qm
if err != nil { }
if err := queue.Add(ctx, log, reqInfo.AccountName, dataFile, qml...); err != nil {
metricSubmission.WithLabelValues("queueerror").Inc() metricSubmission.WithLabelValues("queueerror").Inc()
} }
xcheckf(ctx, err, "adding message to the delivery queue") xcheckf(ctx, err, "adding messages to the delivery queue")
metricSubmission.WithLabelValues("ok").Inc() metricSubmission.WithLabelValues("ok").Inc()
}
var modseq store.ModSeq // Only set if needed. var modseq store.ModSeq // Only set if needed.