mirror of
https://github.com/mjl-/mox.git
synced 2024-12-26 16:33:47 +03:00
5b20cba50a
we don't want external software to include internal details like mlog. slog.Logger is/will be the standard. we still have mlog for its helper functions, and its handler that logs in concise logfmt used by mox. packages that are not meant for reuse still pass around mlog.Log for convenience. we use golang.org/x/exp/slog because we also support the previous Go toolchain version. with the next Go release, we'll switch to the builtin slog.
398 lines
13 KiB
Go
398 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/mjl-/bstore"
|
|
"github.com/mjl-/sconf"
|
|
|
|
"github.com/mjl-/mox/config"
|
|
"github.com/mjl-/mox/dmarcdb"
|
|
"github.com/mjl-/mox/dmarcrpt"
|
|
"github.com/mjl-/mox/dns"
|
|
"github.com/mjl-/mox/mlog"
|
|
"github.com/mjl-/mox/mox-"
|
|
"github.com/mjl-/mox/moxvar"
|
|
"github.com/mjl-/mox/mtasts"
|
|
"github.com/mjl-/mox/mtastsdb"
|
|
"github.com/mjl-/mox/queue"
|
|
"github.com/mjl-/mox/smtp"
|
|
"github.com/mjl-/mox/store"
|
|
"github.com/mjl-/mox/tlsrpt"
|
|
"github.com/mjl-/mox/tlsrptdb"
|
|
)
|
|
|
|
func cmdGentestdata(c *cmd) {
|
|
c.unlisted = true
|
|
c.params = "dest-dir"
|
|
c.help = `Generate a data directory populated, for testing upgrades.`
|
|
args := c.Parse()
|
|
if len(args) != 1 {
|
|
c.Usage()
|
|
}
|
|
|
|
destDataDir, err := filepath.Abs(args[0])
|
|
xcheckf(err, "making destination directory an absolute path")
|
|
|
|
if _, err := os.Stat(destDataDir); err == nil {
|
|
log.Fatalf("destination directory already exists, refusing to generate test data")
|
|
}
|
|
err = os.MkdirAll(destDataDir, 0770)
|
|
xcheckf(err, "creating destination data directory")
|
|
err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
|
|
xcheckf(err, "creating tmp directory")
|
|
|
|
tempfile := func() *os.File {
|
|
f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
|
|
xcheckf(err, "creating temp file")
|
|
return f
|
|
}
|
|
|
|
ctxbg := context.Background()
|
|
mox.Conf.Log[""] = mlog.LevelInfo
|
|
mlog.SetConfig(mox.Conf.Log)
|
|
|
|
const domainsConf = `
|
|
Domains:
|
|
mox.example: nil
|
|
☺.example: nil
|
|
Accounts:
|
|
test0:
|
|
Domain: mox.example
|
|
Destinations:
|
|
test0@mox.example: nil
|
|
test1:
|
|
Domain: mox.example
|
|
Destinations:
|
|
test1@mox.example: nil
|
|
test2:
|
|
Domain: ☺.example
|
|
Destinations:
|
|
☹@☺.example: nil
|
|
JunkFilter:
|
|
Threshold: 0.95
|
|
Params:
|
|
Twograms: true
|
|
MaxPower: 0.1
|
|
TopWords: 10
|
|
IgnoreWords: 0.1
|
|
`
|
|
|
|
mox.ConfigStaticPath = filepath.FromSlash("/tmp/mox-bogus/mox.conf")
|
|
mox.ConfigDynamicPath = filepath.FromSlash("/tmp/mox-bogus/domains.conf")
|
|
mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
|
|
mox.Conf.Static = config.Static{
|
|
DataDir: destDataDir,
|
|
}
|
|
err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
|
|
xcheckf(err, "parsing domains config")
|
|
|
|
const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
|
|
<feedback>
|
|
<report_metadata>
|
|
<org_name>google.com</org_name>
|
|
<email>noreply-dmarc-support@google.com</email>
|
|
<extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
|
|
<report_id>10051505501689795560</report_id>
|
|
<date_range>
|
|
<begin>1596412800</begin>
|
|
<end>1596499199</end>
|
|
</date_range>
|
|
</report_metadata>
|
|
<policy_published>
|
|
<domain>mox.example</domain>
|
|
<adkim>r</adkim>
|
|
<aspf>r</aspf>
|
|
<p>reject</p>
|
|
<sp>reject</sp>
|
|
<pct>100</pct>
|
|
</policy_published>
|
|
<record>
|
|
<row>
|
|
<source_ip>127.0.0.1</source_ip>
|
|
<count>1</count>
|
|
<policy_evaluated>
|
|
<disposition>none</disposition>
|
|
<dkim>pass</dkim>
|
|
<spf>pass</spf>
|
|
</policy_evaluated>
|
|
</row>
|
|
<identifiers>
|
|
<header_from>example.org</header_from>
|
|
</identifiers>
|
|
<auth_results>
|
|
<dkim>
|
|
<domain>example.org</domain>
|
|
<result>pass</result>
|
|
<selector>example</selector>
|
|
</dkim>
|
|
<spf>
|
|
<domain>example.org</domain>
|
|
<result>pass</result>
|
|
</spf>
|
|
</auth_results>
|
|
</record>
|
|
</feedback>
|
|
`
|
|
|
|
const tlsReport = `{
|
|
"organization-name": "Company-X",
|
|
"date-range": {
|
|
"start-datetime": "2016-04-01T00:00:00Z",
|
|
"end-datetime": "2016-04-01T23:59:59Z"
|
|
},
|
|
"contact-info": "sts-reporting@company-x.example",
|
|
"report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
|
|
"policies": [{
|
|
"policy": {
|
|
"policy-type": "sts",
|
|
"policy-string": ["version: STSv1","mode: testing",
|
|
"mx: *.mail.company-y.example","max_age: 86400"],
|
|
"policy-domain": "mox.example",
|
|
"mx-host": ["*.mail.company-y.example"]
|
|
},
|
|
"summary": {
|
|
"total-successful-session-count": 5326,
|
|
"total-failure-session-count": 303
|
|
},
|
|
"failure-details": [{
|
|
"result-type": "certificate-expired",
|
|
"sending-mta-ip": "2001:db8:abcd:0012::1",
|
|
"receiving-mx-hostname": "mx1.mail.company-y.example",
|
|
"failed-session-count": 100
|
|
}, {
|
|
"result-type": "starttls-not-supported",
|
|
"sending-mta-ip": "2001:db8:abcd:0013::1",
|
|
"receiving-mx-hostname": "mx2.mail.company-y.example",
|
|
"receiving-ip": "203.0.113.56",
|
|
"failed-session-count": 200,
|
|
"additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
|
|
}, {
|
|
"result-type": "validation-failure",
|
|
"sending-mta-ip": "198.51.100.62",
|
|
"receiving-ip": "203.0.113.58",
|
|
"receiving-mx-hostname": "mx-backup.mail.company-y.example",
|
|
"failed-session-count": 3,
|
|
"failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
|
|
}]
|
|
}]
|
|
}`
|
|
|
|
err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
|
|
xcheckf(err, "writing moxversion")
|
|
|
|
// Populate dmarc.db.
|
|
err = dmarcdb.Init()
|
|
xcheckf(err, "dmarcdb init")
|
|
report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
|
|
xcheckf(err, "parsing dmarc aggregate report")
|
|
err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
|
|
xcheckf(err, "adding dmarc aggregate report")
|
|
|
|
// Populate mtasts.db.
|
|
err = mtastsdb.Init(false)
|
|
xcheckf(err, "mtastsdb init")
|
|
mtastsPolicy := mtasts.Policy{
|
|
Version: "STSv1",
|
|
Mode: mtasts.ModeTesting,
|
|
MX: []mtasts.STSMX{
|
|
{Domain: dns.Domain{ASCII: "mx1.example.com"}},
|
|
{Domain: dns.Domain{ASCII: "mx2.example.com"}},
|
|
{Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
|
|
},
|
|
MaxAgeSeconds: 1296000,
|
|
}
|
|
err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy, mtastsPolicy.String())
|
|
xcheckf(err, "adding mtastsdb report")
|
|
|
|
// Populate tlsrpt.db.
|
|
err = tlsrptdb.Init()
|
|
xcheckf(err, "tlsrptdb init")
|
|
tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
|
|
xcheckf(err, "parsing tls report")
|
|
err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr)
|
|
xcheckf(err, "adding tls report")
|
|
|
|
// Populate queue, with a message.
|
|
err = queue.Init()
|
|
xcheckf(err, "queue init")
|
|
mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
|
|
rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
|
|
prefix := []byte{}
|
|
mf := tempfile()
|
|
xcheckf(err, "temp file for queue message")
|
|
defer os.Remove(mf.Name())
|
|
defer mf.Close()
|
|
const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
_, err = fmt.Fprint(mf, qmsg)
|
|
xcheckf(err, "writing message")
|
|
qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
|
|
err = queue.Add(ctxbg, c.log, &qm, mf)
|
|
xcheckf(err, "enqueue message")
|
|
|
|
// Create three accounts.
|
|
// First account without messages.
|
|
accTest0, err := store.OpenAccount(c.log, "test0")
|
|
xcheckf(err, "open account test0")
|
|
err = accTest0.ThreadingWait(c.log)
|
|
xcheckf(err, "wait for threading to finish")
|
|
err = accTest0.Close()
|
|
xcheckf(err, "close account")
|
|
|
|
// Second account with one message.
|
|
accTest1, err := store.OpenAccount(c.log, "test1")
|
|
xcheckf(err, "open account test1")
|
|
err = accTest1.ThreadingWait(c.log)
|
|
xcheckf(err, "wait for threading to finish")
|
|
err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
|
|
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m := store.Message{
|
|
MailboxID: inbox.ID,
|
|
MailboxOrigID: inbox.ID,
|
|
MailboxDestinedID: inbox.ID,
|
|
RemoteIP: "1.2.3.4",
|
|
RemoteIPMasked1: "1.2.3.4",
|
|
RemoteIPMasked2: "1.2.3.0",
|
|
RemoteIPMasked3: "1.2.0.0",
|
|
EHLODomain: "other.example",
|
|
MailFrom: "other@remote.example",
|
|
MailFromLocalpart: smtp.Localpart("other"),
|
|
MailFromDomain: "remote.example",
|
|
RcptToLocalpart: "test1",
|
|
RcptToDomain: "mox.example",
|
|
MsgFromLocalpart: "other",
|
|
MsgFromDomain: "remote.example",
|
|
MsgFromOrgDomain: "remote.example",
|
|
EHLOValidated: true,
|
|
MailFromValidated: true,
|
|
MsgFromValidated: true,
|
|
EHLOValidation: store.ValidationStrict,
|
|
MailFromValidation: store.ValidationPass,
|
|
MsgFromValidation: store.ValidationStrict,
|
|
DKIMDomains: []string{"other.example"},
|
|
Size: int64(len(msg)),
|
|
}
|
|
mf := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf, msg)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false)
|
|
|
|
mfname := mf.Name()
|
|
xcheckf(err, "add message to account test1")
|
|
err = mf.Close()
|
|
xcheckf(err, "closing file")
|
|
err = os.Remove(mfname)
|
|
xcheckf(err, "removing temp message file")
|
|
|
|
err = tx.Get(&inbox)
|
|
xcheckf(err, "get inbox")
|
|
inbox.Add(m.MailboxCounts())
|
|
err = tx.Update(&inbox)
|
|
xcheckf(err, "update inbox")
|
|
|
|
return nil
|
|
})
|
|
xcheckf(err, "write transaction with new message")
|
|
err = accTest1.Close()
|
|
xcheckf(err, "close account")
|
|
|
|
// Third account with two messages and junkfilter.
|
|
accTest2, err := store.OpenAccount(c.log, "test2")
|
|
xcheckf(err, "open account test2")
|
|
err = accTest2.ThreadingWait(c.log)
|
|
xcheckf(err, "wait for threading to finish")
|
|
err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
|
|
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m0 := store.Message{
|
|
MailboxID: inbox.ID,
|
|
MailboxOrigID: inbox.ID,
|
|
MailboxDestinedID: inbox.ID,
|
|
RemoteIP: "::1",
|
|
RemoteIPMasked1: "::",
|
|
RemoteIPMasked2: "::",
|
|
RemoteIPMasked3: "::",
|
|
EHLODomain: "other.example",
|
|
MailFrom: "other@remote.example",
|
|
MailFromLocalpart: smtp.Localpart("other"),
|
|
MailFromDomain: "remote.example",
|
|
RcptToLocalpart: "☹",
|
|
RcptToDomain: "☺.example",
|
|
MsgFromLocalpart: "other",
|
|
MsgFromDomain: "remote.example",
|
|
MsgFromOrgDomain: "remote.example",
|
|
EHLOValidated: true,
|
|
MailFromValidated: true,
|
|
MsgFromValidated: true,
|
|
EHLOValidation: store.ValidationStrict,
|
|
MailFromValidation: store.ValidationPass,
|
|
MsgFromValidation: store.ValidationStrict,
|
|
DKIMDomains: []string{"other.example"},
|
|
Size: int64(len(msg0)),
|
|
}
|
|
mf0 := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf0, msg0)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false)
|
|
xcheckf(err, "add message to account test2")
|
|
|
|
mf0name := mf0.Name()
|
|
err = mf0.Close()
|
|
xcheckf(err, "closing file")
|
|
err = os.Remove(mf0name)
|
|
xcheckf(err, "removing temp message file")
|
|
|
|
err = tx.Get(&inbox)
|
|
xcheckf(err, "get inbox")
|
|
inbox.Add(m0.MailboxCounts())
|
|
err = tx.Update(&inbox)
|
|
xcheckf(err, "update inbox")
|
|
|
|
sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const prefix1 = "Extra: test\r\n"
|
|
const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m1 := store.Message{
|
|
MailboxID: sent.ID,
|
|
MailboxOrigID: sent.ID,
|
|
MailboxDestinedID: sent.ID,
|
|
Flags: store.Flags{Seen: true, Junk: true},
|
|
Size: int64(len(prefix1) + len(msg1)),
|
|
MsgPrefix: []byte(prefix1),
|
|
}
|
|
mf1 := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf1, msg1)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false)
|
|
xcheckf(err, "add message to account test2")
|
|
|
|
mf1name := mf1.Name()
|
|
err = mf1.Close()
|
|
xcheckf(err, "closing file")
|
|
err = os.Remove(mf1name)
|
|
xcheckf(err, "removing temp message file")
|
|
|
|
err = tx.Get(&sent)
|
|
xcheckf(err, "get sent")
|
|
sent.Add(m1.MailboxCounts())
|
|
err = tx.Update(&sent)
|
|
xcheckf(err, "update sent")
|
|
|
|
return nil
|
|
})
|
|
xcheckf(err, "write transaction with new message")
|
|
err = accTest2.Close()
|
|
xcheckf(err, "close account")
|
|
}
|