mirror of
https://github.com/mjl-/mox.git
synced 2024-12-26 00:13:47 +03:00
aad5a5bcb9
the backup command will make consistent snapshots of all the database files. i had been copying the db files before, and it usually works. but if the file is modified during the backup, it is inconsistent and is likely to generate errors when reading (can be at any moment in the future, when reading some db page). "mox backup" opens the database file and writes out a copy in a transaction. it also duplicates the message files. before doing a restore, you could run "mox verifydata" on the to-be-restored "data" directory. it check the database files, and compares the message files with the database. the new "gentestdata" subcommand generates a basic "data" directory, with a queue and a few accounts. we will use it in the future along with "verifydata" to test upgrades from old version to the latest version. both when going to the next version, and when skipping several versions. the script test-upgrades.sh executes these tests and doesn't do anything at the moment, because no releases have this subcommand yet. inspired by a failed upgrade attempt of a pre-release version.
361 lines
12 KiB
Go
361 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/mjl-/bstore"
|
|
"github.com/mjl-/sconf"
|
|
|
|
"github.com/mjl-/mox/config"
|
|
"github.com/mjl-/mox/dmarcdb"
|
|
"github.com/mjl-/mox/dmarcrpt"
|
|
"github.com/mjl-/mox/dns"
|
|
"github.com/mjl-/mox/mlog"
|
|
"github.com/mjl-/mox/mox-"
|
|
"github.com/mjl-/mox/moxvar"
|
|
"github.com/mjl-/mox/mtasts"
|
|
"github.com/mjl-/mox/mtastsdb"
|
|
"github.com/mjl-/mox/queue"
|
|
"github.com/mjl-/mox/smtp"
|
|
"github.com/mjl-/mox/store"
|
|
"github.com/mjl-/mox/tlsrpt"
|
|
"github.com/mjl-/mox/tlsrptdb"
|
|
)
|
|
|
|
func cmdGentestdata(c *cmd) {
|
|
c.unlisted = true
|
|
c.params = "dest-dir"
|
|
c.help = `Generate a data directory populated, for testing upgrades.`
|
|
args := c.Parse()
|
|
if len(args) != 1 {
|
|
c.Usage()
|
|
}
|
|
|
|
destDataDir, err := filepath.Abs(args[0])
|
|
xcheckf(err, "making destination directory an absolute path")
|
|
|
|
if _, err := os.Stat(destDataDir); err == nil {
|
|
log.Fatalf("destination directory already exists, refusing to generate test data")
|
|
}
|
|
err = os.MkdirAll(destDataDir, 0770)
|
|
xcheckf(err, "creating destination data directory")
|
|
err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
|
|
xcheckf(err, "creating tmp directory")
|
|
|
|
tempfile := func() *os.File {
|
|
f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
|
|
xcheckf(err, "creating temp file")
|
|
return f
|
|
}
|
|
|
|
ctxbg := context.Background()
|
|
mox.Shutdown = ctxbg
|
|
mox.Context = ctxbg
|
|
mox.Conf.Log[""] = mlog.LevelInfo
|
|
mlog.SetConfig(mox.Conf.Log)
|
|
|
|
const domainsConf = `
|
|
Domains:
|
|
mox.example: nil
|
|
☺.example: nil
|
|
Accounts:
|
|
test0:
|
|
Domain: mox.example
|
|
Destinations:
|
|
test0@mox.example: nil
|
|
test1:
|
|
Domain: mox.example
|
|
Destinations:
|
|
test1@mox.example: nil
|
|
test2:
|
|
Domain: ☺.example
|
|
Destinations:
|
|
☹@☺.example: nil
|
|
JunkFilter:
|
|
Threshold: 0.95
|
|
Params:
|
|
Twograms: true
|
|
MaxPower: 0.1
|
|
TopWords: 10
|
|
IgnoreWords: 0.1
|
|
`
|
|
|
|
mox.ConfigStaticPath = "/tmp/mox-bogus/mox.conf"
|
|
mox.ConfigDynamicPath = "/tmp/mox-bogus/domains.conf"
|
|
mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
|
|
mox.Conf.Static = config.Static{
|
|
DataDir: destDataDir,
|
|
}
|
|
err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
|
|
xcheckf(err, "parsing domains config")
|
|
|
|
const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
|
|
<feedback>
|
|
<report_metadata>
|
|
<org_name>google.com</org_name>
|
|
<email>noreply-dmarc-support@google.com</email>
|
|
<extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
|
|
<report_id>10051505501689795560</report_id>
|
|
<date_range>
|
|
<begin>1596412800</begin>
|
|
<end>1596499199</end>
|
|
</date_range>
|
|
</report_metadata>
|
|
<policy_published>
|
|
<domain>mox.example</domain>
|
|
<adkim>r</adkim>
|
|
<aspf>r</aspf>
|
|
<p>reject</p>
|
|
<sp>reject</sp>
|
|
<pct>100</pct>
|
|
</policy_published>
|
|
<record>
|
|
<row>
|
|
<source_ip>127.0.0.1</source_ip>
|
|
<count>1</count>
|
|
<policy_evaluated>
|
|
<disposition>none</disposition>
|
|
<dkim>pass</dkim>
|
|
<spf>pass</spf>
|
|
</policy_evaluated>
|
|
</row>
|
|
<identifiers>
|
|
<header_from>example.org</header_from>
|
|
</identifiers>
|
|
<auth_results>
|
|
<dkim>
|
|
<domain>example.org</domain>
|
|
<result>pass</result>
|
|
<selector>example</selector>
|
|
</dkim>
|
|
<spf>
|
|
<domain>example.org</domain>
|
|
<result>pass</result>
|
|
</spf>
|
|
</auth_results>
|
|
</record>
|
|
</feedback>
|
|
`
|
|
|
|
const tlsReport = `{
|
|
"organization-name": "Company-X",
|
|
"date-range": {
|
|
"start-datetime": "2016-04-01T00:00:00Z",
|
|
"end-datetime": "2016-04-01T23:59:59Z"
|
|
},
|
|
"contact-info": "sts-reporting@company-x.example",
|
|
"report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
|
|
"policies": [{
|
|
"policy": {
|
|
"policy-type": "sts",
|
|
"policy-string": ["version: STSv1","mode: testing",
|
|
"mx: *.mail.company-y.example","max_age: 86400"],
|
|
"policy-domain": "mox.example",
|
|
"mx-host": ["*.mail.company-y.example"]
|
|
},
|
|
"summary": {
|
|
"total-successful-session-count": 5326,
|
|
"total-failure-session-count": 303
|
|
},
|
|
"failure-details": [{
|
|
"result-type": "certificate-expired",
|
|
"sending-mta-ip": "2001:db8:abcd:0012::1",
|
|
"receiving-mx-hostname": "mx1.mail.company-y.example",
|
|
"failed-session-count": 100
|
|
}, {
|
|
"result-type": "starttls-not-supported",
|
|
"sending-mta-ip": "2001:db8:abcd:0013::1",
|
|
"receiving-mx-hostname": "mx2.mail.company-y.example",
|
|
"receiving-ip": "203.0.113.56",
|
|
"failed-session-count": 200,
|
|
"additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
|
|
}, {
|
|
"result-type": "validation-failure",
|
|
"sending-mta-ip": "198.51.100.62",
|
|
"receiving-ip": "203.0.113.58",
|
|
"receiving-mx-hostname": "mx-backup.mail.company-y.example",
|
|
"failed-session-count": 3,
|
|
"failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
|
|
}]
|
|
}]
|
|
}`
|
|
|
|
err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
|
|
xcheckf(err, "writing moxversion")
|
|
|
|
// Populate dmarc.db.
|
|
err = dmarcdb.Init()
|
|
xcheckf(err, "dmarcdb init")
|
|
report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
|
|
xcheckf(err, "parsing dmarc report")
|
|
err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
|
|
xcheckf(err, "adding dmarc report")
|
|
|
|
// Populate mtasts.db.
|
|
err = mtastsdb.Init(false)
|
|
xcheckf(err, "mtastsdb init")
|
|
mtastsPolicy := mtasts.Policy{
|
|
Version: "STSv1",
|
|
Mode: mtasts.ModeTesting,
|
|
MX: []mtasts.STSMX{
|
|
{Domain: dns.Domain{ASCII: "mx1.example.com"}},
|
|
{Domain: dns.Domain{ASCII: "mx2.example.com"}},
|
|
{Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
|
|
},
|
|
MaxAgeSeconds: 1296000,
|
|
}
|
|
err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy)
|
|
xcheckf(err, "adding mtastsdb report")
|
|
|
|
// Populate tlsrpt.db.
|
|
err = tlsrptdb.Init()
|
|
xcheckf(err, "tlsrptdb init")
|
|
tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
|
|
xcheckf(err, "parsing tls report")
|
|
err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", tlsr)
|
|
xcheckf(err, "adding tls report")
|
|
|
|
// Populate queue, with a message.
|
|
err = queue.Init()
|
|
xcheckf(err, "queue init")
|
|
mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
|
|
rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
|
|
prefix := []byte{}
|
|
mf := tempfile()
|
|
xcheckf(err, "temp file for queue message")
|
|
defer mf.Close()
|
|
const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
_, err = fmt.Fprint(mf, qmsg)
|
|
xcheckf(err, "writing message")
|
|
err = queue.Add(ctxbg, mlog.New("gentestdata"), "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), prefix, mf, nil, true)
|
|
xcheckf(err, "enqueue message")
|
|
|
|
// Create three accounts.
|
|
// First account without messages.
|
|
accTest0, err := store.OpenAccount("test0")
|
|
xcheckf(err, "open account test0")
|
|
err = accTest0.Close()
|
|
xcheckf(err, "close account")
|
|
|
|
// Second account with one message.
|
|
accTest1, err := store.OpenAccount("test1")
|
|
xcheckf(err, "open account test1")
|
|
err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
|
|
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m := store.Message{
|
|
MailboxID: inbox.ID,
|
|
MailboxOrigID: inbox.ID,
|
|
MailboxDestinedID: inbox.ID,
|
|
RemoteIP: "1.2.3.4",
|
|
RemoteIPMasked1: "1.2.3.4",
|
|
RemoteIPMasked2: "1.2.3.0",
|
|
RemoteIPMasked3: "1.2.0.0",
|
|
EHLODomain: "other.example",
|
|
MailFrom: "other@remote.example",
|
|
MailFromLocalpart: smtp.Localpart("other"),
|
|
MailFromDomain: "remote.example",
|
|
RcptToLocalpart: "test1",
|
|
RcptToDomain: "mox.example",
|
|
MsgFromLocalpart: "other",
|
|
MsgFromDomain: "remote.example",
|
|
MsgFromOrgDomain: "remote.example",
|
|
EHLOValidated: true,
|
|
MailFromValidated: true,
|
|
MsgFromValidated: true,
|
|
EHLOValidation: store.ValidationStrict,
|
|
MailFromValidation: store.ValidationPass,
|
|
MsgFromValidation: store.ValidationStrict,
|
|
DKIMDomains: []string{"other.example"},
|
|
Size: int64(len(msg)),
|
|
}
|
|
mf := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf, msg)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest1.DeliverMessage(mlog.New("gentestdata"), tx, &m, mf, true, false, false, true)
|
|
xcheckf(err, "add message to account test1")
|
|
err = mf.Close()
|
|
xcheckf(err, "closing file")
|
|
return nil
|
|
})
|
|
xcheckf(err, "write transaction with new message")
|
|
err = accTest1.Close()
|
|
xcheckf(err, "close account")
|
|
|
|
// Third account with two messages and junkfilter.
|
|
accTest2, err := store.OpenAccount("test2")
|
|
xcheckf(err, "open account test2")
|
|
err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
|
|
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m0 := store.Message{
|
|
MailboxID: inbox.ID,
|
|
MailboxOrigID: inbox.ID,
|
|
MailboxDestinedID: inbox.ID,
|
|
RemoteIP: "::1",
|
|
RemoteIPMasked1: "::",
|
|
RemoteIPMasked2: "::",
|
|
RemoteIPMasked3: "::",
|
|
EHLODomain: "other.example",
|
|
MailFrom: "other@remote.example",
|
|
MailFromLocalpart: smtp.Localpart("other"),
|
|
MailFromDomain: "remote.example",
|
|
RcptToLocalpart: "☹",
|
|
RcptToDomain: "☺.example",
|
|
MsgFromLocalpart: "other",
|
|
MsgFromDomain: "remote.example",
|
|
MsgFromOrgDomain: "remote.example",
|
|
EHLOValidated: true,
|
|
MailFromValidated: true,
|
|
MsgFromValidated: true,
|
|
EHLOValidation: store.ValidationStrict,
|
|
MailFromValidation: store.ValidationPass,
|
|
MsgFromValidation: store.ValidationStrict,
|
|
DKIMDomains: []string{"other.example"},
|
|
Size: int64(len(msg0)),
|
|
}
|
|
mf0 := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf0, msg0)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m0, mf0, true, false, false, false)
|
|
xcheckf(err, "add message to account test2")
|
|
err = mf0.Close()
|
|
xcheckf(err, "closing file")
|
|
|
|
sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
|
|
xcheckf(err, "looking up inbox")
|
|
const prefix1 = "Extra: test\r\n"
|
|
const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
|
|
m1 := store.Message{
|
|
MailboxID: sent.ID,
|
|
MailboxOrigID: sent.ID,
|
|
MailboxDestinedID: sent.ID,
|
|
Flags: store.Flags{Seen: true, Junk: true},
|
|
Size: int64(len(prefix1) + len(msg1)),
|
|
MsgPrefix: []byte(prefix),
|
|
}
|
|
mf1 := tempfile()
|
|
xcheckf(err, "creating temp file for delivery")
|
|
_, err = fmt.Fprint(mf1, msg1)
|
|
xcheckf(err, "writing deliver message to file")
|
|
err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m1, mf1, true, true, false, false)
|
|
xcheckf(err, "add message to account test2")
|
|
err = mf1.Close()
|
|
xcheckf(err, "closing file")
|
|
|
|
return nil
|
|
})
|
|
xcheckf(err, "write transaction with new message")
|
|
err = accTest2.Close()
|
|
xcheckf(err, "close account")
|
|
}
|