mirror of
https://github.com/mjl-/mox.git
synced 2024-12-26 16:33:47 +03:00
d73bda7511
so a single user cannot fill up the disk. by default, there is (still) no limit. a default can be set in the config file for all accounts, and a per-account max size can be set that would override any global setting. this does not take into account disk usage of the index database. and also not of any file system overhead.
398 lines
13 KiB
Go
398 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/mjl-/bstore"
|
|
"github.com/mjl-/sconf"
|
|
|
|
"github.com/mjl-/mox/config"
|
|
"github.com/mjl-/mox/dmarcdb"
|
|
"github.com/mjl-/mox/dmarcrpt"
|
|
"github.com/mjl-/mox/dns"
|
|
"github.com/mjl-/mox/mlog"
|
|
"github.com/mjl-/mox/mox-"
|
|
"github.com/mjl-/mox/moxvar"
|
|
"github.com/mjl-/mox/mtasts"
|
|
"github.com/mjl-/mox/mtastsdb"
|
|
"github.com/mjl-/mox/queue"
|
|
"github.com/mjl-/mox/smtp"
|
|
"github.com/mjl-/mox/store"
|
|
"github.com/mjl-/mox/tlsrpt"
|
|
"github.com/mjl-/mox/tlsrptdb"
|
|
)
|
|
|
|
func cmdGentestdata(c *cmd) {
|
|
c.unlisted = true
|
|
c.params = "dest-dir"
|
|
c.help = `Generate a data directory populated, for testing upgrades.`
|
|
args := c.Parse()
|
|
if len(args) != 1 {
|
|
c.Usage()
|
|
}
|
|
|
|
destDataDir, err := filepath.Abs(args[0])
|
|
xcheckf(err, "making destination directory an absolute path")
|
|
|
|
if _, err := os.Stat(destDataDir); err == nil {
|
|
log.Fatalf("destination directory already exists, refusing to generate test data")
|
|
}
|
|
err = os.MkdirAll(destDataDir, 0770)
|
|
xcheckf(err, "creating destination data directory")
|
|
err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
|
|
xcheckf(err, "creating tmp directory")
|
|
|
|
tempfile := func() *os.File {
|
|
f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
|
|
xcheckf(err, "creating temp file")
|
|
return f
|
|
}
|
|
|
|
ctxbg := context.Background()
|
|
mox.Conf.Log[""] = mlog.LevelInfo
|
|
mlog.SetConfig(mox.Conf.Log)
|
|
|
|
const domainsConf = `
|
|
Domains:
|
|
mox.example: nil
|
|
☺.example: nil
|
|
Accounts:
|
|
test0:
|
|
Domain: mox.example
|
|
Destinations:
|
|
test0@mox.example: nil
|
|
test1:
|
|
Domain: mox.example
|
|
Destinations:
|
|
test1@mox.example: nil
|
|
test2:
|
|
Domain: ☺.example
|
|
Destinations:
|
|
☹@☺.example: nil
|
|
JunkFilter:
|
|
Threshold: 0.95
|
|
Params:
|
|
Twograms: true
|
|
MaxPower: 0.1
|
|
TopWords: 10
|
|
IgnoreWords: 0.1
|
|
`
|
|
|
|
mox.ConfigStaticPath = filepath.FromSlash("/tmp/mox-bogus/mox.conf")
|
|
mox.ConfigDynamicPath = filepath.FromSlash("/tmp/mox-bogus/domains.conf")
|
|
mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
|
|
mox.Conf.Static = config.Static{
|
|
DataDir: destDataDir,
|
|
}
|
|
err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
|
|
xcheckf(err, "parsing domains config")
|
|
|
|
const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
|
|
<feedback>
|
|
<report_metadata>
|
|
<org_name>google.com</org_name>
|
|
<email>noreply-dmarc-support@google.com</email>
|
|
<extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
|
|
<report_id>10051505501689795560</report_id>
|
|
<date_range>
|
|
<begin>1596412800</begin>
|
|
<end>1596499199</end>
|
|
</date_range>
|
|
</report_metadata>
|
|
<policy_published>
|
|
<domain>mox.example</domain>
|
|
<adkim>r</adkim>
|
|
<aspf>r</aspf>
|
|
<p>reject</p>
|
|
<sp>reject</sp>
|
|
<pct>100</pct>
|
|
</policy_published>
|
|
<record>
|
|
<row>
|
|
<source_ip>127.0.0.1</source_ip>
|
|
<count>1</count>
|
|
<policy_evaluated>
|
|
<disposition>none</disposition>
|
|
<dkim>pass</dkim>
|
|
<spf>pass</spf>
|
|
</policy_evaluated>
|
|
</row>
|
|
<identifiers>
|
|
<header_from>example.org</header_from>
|
|
</identifiers>
|
|
<auth_results>
|
|
<dkim>
|
|
<domain>example.org</domain>
|
|
<result>pass</result>
|
|
<selector>example</selector>
|
|
</dkim>
|
|
<spf>
|
|
<domain>example.org</domain>
|
|
<result>pass</result>
|
|
</spf>
|
|
</auth_results>
|
|
</record>
|
|
</feedback>
|
|
`
|
|
|
|
const tlsReport = `{
|
|
"organization-name": "Company-X",
|
|
"date-range": {
|
|
"start-datetime": "2016-04-01T00:00:00Z",
|
|
"end-datetime": "2016-04-01T23:59:59Z"
|
|
},
|
|
"contact-info": "sts-reporting@company-x.example",
|
|
"report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
|
|
"policies": [{
|
|
"policy": {
|
|
"policy-type": "sts",
|
|
"policy-string": ["version: STSv1","mode: testing",
|
|
"mx: *.mail.company-y.example","max_age: 86400"],
|
|
"policy-domain": "mox.example",
|
|
"mx-host": ["*.mail.company-y.example"]
|
|
},
|
|
"summary": {
|
|
"total-successful-session-count": 5326,
|
|
"total-failure-session-count": 303
|
|
},
|
|
"failure-details": [{
|
|
"result-type": "certificate-expired",
|
|
"sending-mta-ip": "2001:db8:abcd:0012::1",
|
|
"receiving-mx-hostname": "mx1.mail.company-y.example",
|
|
"failed-session-count": 100
|
|
}, {
|
|
"result-type": "starttls-not-supported",
|
|
"sending-mta-ip": "2001:db8:abcd:0013::1",
|
|
"receiving-mx-hostname": "mx2.mail.company-y.example",
|
|
"receiving-ip": "203.0.113.56",
|
|
"failed-session-count": 200,
|
|
"additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
|
|
}, {
|
|
"result-type": "validation-failure",
|
|
"sending-mta-ip": "198.51.100.62",
|
|
"receiving-ip": "203.0.113.58",
|
|
"receiving-mx-hostname": "mx-backup.mail.company-y.example",
|
|
"failed-session-count": 3,
|
|
"failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
|
|
}]
|
|
}]
|
|
}`
|
|
|
|
err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
|
|
xcheckf(err, "writing moxversion")
|
|
|
|
// Populate dmarc.db.
|
|
err = dmarcdb.Init()
|
|
xcheckf(err, "dmarcdb init")
|
|
report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
|
|
xcheckf(err, "parsing dmarc aggregate report")
|
|
err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
|
|
xcheckf(err, "adding dmarc aggregate report")
|
|
|
|
// Populate mtasts.db.
|
|
err = mtastsdb.Init(false)
|
|
xcheckf(err, "mtastsdb init")
|
|
mtastsPolicy := mtasts.Policy{
|
|
Version: "STSv1",
|
|
Mode: mtasts.ModeTesting,
|
|
MX: []mtasts.STSMX{
|
|
{Domain: dns.Domain{ASCII: "mx1.example.com"}},
|
|
{Domain: dns.Domain{ASCII: "mx2.example.com"}},
|
|
{Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
|
|
},
|
|
MaxAgeSeconds: 1296000,
|
|
}
|
|
err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy, mtastsPolicy.String())
|
|
xcheckf(err, "adding mtastsdb report")
|
|
|
|
// Populate tlsrpt.db.
|
|
err = tlsrptdb.Init()
|
|
xcheckf(err, "tlsrptdb init")
|
|
tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
|
|
xcheckf(err, "parsing tls report")
|
|
err = tlsrptdb.AddReport(ctxbg, c.log, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", false, tlsr)
|
|
xcheckf(err, "adding tls report")
|
|
|
|
// Populate queue, with a message.
|
|
err = queue.Init()
|
|
xcheckf(err, "queue init")
|
|
mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
|
|
rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
|
|
prefix := []byte{}
|
|
mf := tempfile()
|
|
xcheckf(err, "temp file for queue message")
|
|
defer os.Remove(mf.Name())
|
|
defer mf.Close()
|
|
const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
_, err = fmt.Fprint(mf, qmsg)
|
|
xcheckf(err, "writing message")
|
|
qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
|
|
err = queue.Add(ctxbg, c.log, &qm, mf)
|
|
xcheckf(err, "enqueue message")
|
|
|
|
// Create three accounts.
|
|
// First account without messages.
|
|
accTest0, err := store.OpenAccount(c.log, "test0")
|
|
xcheckf(err, "open account test0")
|
|
err = accTest0.ThreadingWait(c.log)
|
|
xcheckf(err, "wait for threading to finish")
|
|
err = accTest0.Close()
|
|
xcheckf(err, "close account")
|
|
|
|
// Second account with one message.
|
|
accTest1, err := store.OpenAccount(c.log, "test1")
|
|
xcheckf(err, "open account test1")
|
|
err = accTest1.ThreadingWait(c.log)
|
|
xcheckf(err, "wait for threading to finish")
|
|
err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
|
|
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m := store.Message{
|
|
MailboxID: inbox.ID,
|
|
MailboxOrigID: inbox.ID,
|
|
MailboxDestinedID: inbox.ID,
|
|
RemoteIP: "1.2.3.4",
|
|
RemoteIPMasked1: "1.2.3.4",
|
|
RemoteIPMasked2: "1.2.3.0",
|
|
RemoteIPMasked3: "1.2.0.0",
|
|
EHLODomain: "other.example",
|
|
MailFrom: "other@remote.example",
|
|
MailFromLocalpart: smtp.Localpart("other"),
|
|
MailFromDomain: "remote.example",
|
|
RcptToLocalpart: "test1",
|
|
RcptToDomain: "mox.example",
|
|
MsgFromLocalpart: "other",
|
|
MsgFromDomain: "remote.example",
|
|
MsgFromOrgDomain: "remote.example",
|
|
EHLOValidated: true,
|
|
MailFromValidated: true,
|
|
MsgFromValidated: true,
|
|
EHLOValidation: store.ValidationStrict,
|
|
MailFromValidation: store.ValidationPass,
|
|
MsgFromValidation: store.ValidationStrict,
|
|
DKIMDomains: []string{"other.example"},
|
|
Size: int64(len(msg)),
|
|
}
|
|
mf := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf, msg)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest1.DeliverMessage(c.log, tx, &m, mf, false, true, false, true)
|
|
|
|
mfname := mf.Name()
|
|
xcheckf(err, "add message to account test1")
|
|
err = mf.Close()
|
|
xcheckf(err, "closing file")
|
|
err = os.Remove(mfname)
|
|
xcheckf(err, "removing temp message file")
|
|
|
|
err = tx.Get(&inbox)
|
|
xcheckf(err, "get inbox")
|
|
inbox.Add(m.MailboxCounts())
|
|
err = tx.Update(&inbox)
|
|
xcheckf(err, "update inbox")
|
|
|
|
return nil
|
|
})
|
|
xcheckf(err, "write transaction with new message")
|
|
err = accTest1.Close()
|
|
xcheckf(err, "close account")
|
|
|
|
// Third account with two messages and junkfilter.
|
|
accTest2, err := store.OpenAccount(c.log, "test2")
|
|
xcheckf(err, "open account test2")
|
|
err = accTest2.ThreadingWait(c.log)
|
|
xcheckf(err, "wait for threading to finish")
|
|
err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
|
|
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m0 := store.Message{
|
|
MailboxID: inbox.ID,
|
|
MailboxOrigID: inbox.ID,
|
|
MailboxDestinedID: inbox.ID,
|
|
RemoteIP: "::1",
|
|
RemoteIPMasked1: "::",
|
|
RemoteIPMasked2: "::",
|
|
RemoteIPMasked3: "::",
|
|
EHLODomain: "other.example",
|
|
MailFrom: "other@remote.example",
|
|
MailFromLocalpart: smtp.Localpart("other"),
|
|
MailFromDomain: "remote.example",
|
|
RcptToLocalpart: "☹",
|
|
RcptToDomain: "☺.example",
|
|
MsgFromLocalpart: "other",
|
|
MsgFromDomain: "remote.example",
|
|
MsgFromOrgDomain: "remote.example",
|
|
EHLOValidated: true,
|
|
MailFromValidated: true,
|
|
MsgFromValidated: true,
|
|
EHLOValidation: store.ValidationStrict,
|
|
MailFromValidation: store.ValidationPass,
|
|
MsgFromValidation: store.ValidationStrict,
|
|
DKIMDomains: []string{"other.example"},
|
|
Size: int64(len(msg0)),
|
|
}
|
|
mf0 := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf0, msg0)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest2.DeliverMessage(c.log, tx, &m0, mf0, false, false, false, true)
|
|
xcheckf(err, "add message to account test2")
|
|
|
|
mf0name := mf0.Name()
|
|
err = mf0.Close()
|
|
xcheckf(err, "closing file")
|
|
err = os.Remove(mf0name)
|
|
xcheckf(err, "removing temp message file")
|
|
|
|
err = tx.Get(&inbox)
|
|
xcheckf(err, "get inbox")
|
|
inbox.Add(m0.MailboxCounts())
|
|
err = tx.Update(&inbox)
|
|
xcheckf(err, "update inbox")
|
|
|
|
sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const prefix1 = "Extra: test\r\n"
|
|
const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m1 := store.Message{
|
|
MailboxID: sent.ID,
|
|
MailboxOrigID: sent.ID,
|
|
MailboxDestinedID: sent.ID,
|
|
Flags: store.Flags{Seen: true, Junk: true},
|
|
Size: int64(len(prefix1) + len(msg1)),
|
|
MsgPrefix: []byte(prefix1),
|
|
}
|
|
mf1 := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf1, msg1)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest2.DeliverMessage(c.log, tx, &m1, mf1, false, false, false, true)
|
|
xcheckf(err, "add message to account test2")
|
|
|
|
mf1name := mf1.Name()
|
|
err = mf1.Close()
|
|
xcheckf(err, "closing file")
|
|
err = os.Remove(mf1name)
|
|
xcheckf(err, "removing temp message file")
|
|
|
|
err = tx.Get(&sent)
|
|
xcheckf(err, "get sent")
|
|
sent.Add(m1.MailboxCounts())
|
|
err = tx.Update(&sent)
|
|
xcheckf(err, "update sent")
|
|
|
|
return nil
|
|
})
|
|
xcheckf(err, "write transaction with new message")
|
|
err = accTest2.Close()
|
|
xcheckf(err, "close account")
|
|
}
|