From 5817e87a3228df9a24c9b65dd5aaac3456ce380d Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Sat, 1 Jul 2023 16:43:20 +0200 Subject: [PATCH] add subcommand "ximport", that is like "import" but directly access files in the datadir so mox doesn't have to be running when you run it. will be useful for testing in the near future. this also moves cpuprof and memprof cli flags to top-level flag parsing, so all commands can use them. --- import.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ junk.go | 47 ---------------------------------- main.go | 9 +++++++ profile.go | 45 ++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 47 deletions(-) create mode 100644 profile.go diff --git a/import.go b/import.go index df2f978..2905da6 100644 --- a/import.go +++ b/import.go @@ -1,12 +1,14 @@ package main import ( + "bufio" "context" "encoding/json" "errors" "fmt" "io" "log" + "net" "os" "path/filepath" "runtime/debug" @@ -15,9 +17,11 @@ import ( "golang.org/x/exp/maps" + "github.com/mjl-/mox/config" "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) @@ -89,6 +93,77 @@ func xcmdImport(mbox bool, args []string, c *cmd) { ctl := xctl() ctl.xwrite(ctlcmd) + xcmdImportCtl(ctl, account, mailbox, src) +} + +func cmdXImportMaildir(c *cmd) { + c.unlisted = true + c.params = "accountdir mailboxname maildir" + c.help = `Import a maildir into an account by directly accessing the data directory. + + +See "mox help import maildir" for details. +` + xcmdXImport(false, c) +} + +func cmdXImportMbox(c *cmd) { + c.unlisted = true + c.params = "accountdir mailboxname mbox" + c.help = `Import an mbox into an account by directly accessing the data directory. + +See "mox help import mbox" for details. +` + xcmdXImport(true, c) +} + +func xcmdXImport(mbox bool, c *cmd) { + args := c.Parse() + if len(args) != 3 { + c.Usage() + } + + accountdir := args[0] + mailbox := args[1] + if strings.EqualFold(mailbox, "inbox") { + mailbox = "Inbox" + } + src := args[2] + + var ctlcmd string + if mbox { + ctlcmd = "importmbox" + } else { + ctlcmd = "importmaildir" + } + + account := filepath.Base(accountdir) + + // Set up the mox config so the account can be opened. + if filepath.Base(filepath.Dir(accountdir)) != "accounts" { + log.Fatalf("accountdir must be of the form .../accounts/") + } + var err error + mox.Conf.Static.DataDir, err = filepath.Abs(filepath.Dir(filepath.Dir(accountdir))) + xcheckf(err, "making absolute datadir") + mox.ConfigStaticPath = "fake.conf" + mox.Conf.DynamicLastCheck = time.Now().Add(time.Hour) // Silence errors about config file. + mox.Conf.Dynamic.Accounts = map[string]config.Account{ + account: {}, + } + switchDone := store.Switchboard() + defer close(switchDone) + + xlog := mlog.New("import") + cconn, sconn := net.Pipe() + clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: xlog} + serverctl := ctl{cmd: ctlcmd, conn: sconn, r: bufio.NewReader(sconn), log: xlog} + go importctl(context.Background(), &serverctl, true) + + xcmdImportCtl(&clientctl, account, mailbox, src) +} + +func xcmdImportCtl(ctl *ctl, account, mailbox, src string) { ctl.xwrite(account) ctl.xwrite(mailbox) ctl.xwrite(src) diff --git a/junk.go b/junk.go index 277fbc4..83ba273 100644 --- a/junk.go +++ b/junk.go @@ -20,8 +20,6 @@ import ( "log" mathrand "math/rand" "os" - "runtime" - "runtime/pprof" "sort" "time" @@ -33,7 +31,6 @@ import ( type junkArgs struct { params junk.Params - cpuprofile, memprofile string spamThreshold float64 trainRatio float64 seed bool @@ -42,43 +39,6 @@ type junkArgs struct { debug bool } -func (a junkArgs) Memprofile() { - if a.memprofile == "" { - return - } - - f, err := os.Create(a.memprofile) - xcheckf(err, "creating memory profile") - defer func() { - if err := f.Close(); err != nil { - log.Printf("closing memory profile: %v", err) - } - }() - runtime.GC() // get up-to-date statistics - err = pprof.WriteHeapProfile(f) - xcheckf(err, "writing memory profile") -} - -func (a junkArgs) Profile() func() { - if a.cpuprofile == "" { - return func() { - a.Memprofile() - } - } - - f, err := os.Create(a.cpuprofile) - xcheckf(err, "creating CPU profile") - err = pprof.StartCPUProfile(f) - xcheckf(err, "start CPU profile") - return func() { - pprof.StopCPUProfile() - if err := f.Close(); err != nil { - log.Printf("closing cpu profile: %v", err) - } - a.Memprofile() - } -} - func (a junkArgs) SetLogLevel() { mox.Conf.Log[""] = mlog.LevelInfo if a.debug { @@ -104,8 +64,6 @@ func junkFlags(fs *flag.FlagSet) (a junkArgs) { fs.StringVar(&a.databasePath, "dbpath", "filter.db", "database file for ham/spam words") fs.StringVar(&a.bloomfilterPath, "bloompath", "filter.bloom", "bloom filter for ignoring unique strings") - fs.StringVar(&a.cpuprofile, "cpuprof", "", "store cpu profile to file") - fs.StringVar(&a.memprofile, "memprof", "", "store mem profile to file") return } @@ -132,7 +90,6 @@ func cmdJunkTrain(c *cmd) { if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath)) @@ -162,7 +119,6 @@ func cmdJunkCheck(c *cmd) { if len(args) != 1 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false)) @@ -187,7 +143,6 @@ func cmdJunkTest(c *cmd) { if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false)) @@ -244,7 +199,6 @@ messages are shuffled, with optional random seed.` if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath)) @@ -336,7 +290,6 @@ func cmdJunkPlay(c *cmd) { if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath)) diff --git a/main.go b/main.go index 543268a..2c659ae 100644 --- a/main.go +++ b/main.go @@ -152,6 +152,8 @@ var commands = []struct { {"updates serve", cmdUpdatesServe}, {"updates verify", cmdUpdatesVerify}, {"gentestdata", cmdGentestdata}, + {"ximport maildir", cmdXImportMaildir}, + {"ximport mbox", cmdXImportMbox}, } var cmds []cmd @@ -391,12 +393,19 @@ func main() { flag.StringVar(&loglevel, "loglevel", "", "if non-empty, this log level is set early in startup") flag.BoolVar(&pedantic, "pedantic", false, "protocol violations result in errors instead of accepting/working around them") + var cpuprofile, memprofile string + flag.StringVar(&cpuprofile, "cpuprof", "", "store cpu profile to file") + flag.StringVar(&memprofile, "memprof", "", "store mem profile to file") + flag.Usage = func() { usage(cmds, false) } flag.Parse() args := flag.Args() if len(args) == 0 { usage(cmds, false) } + + defer profile(cpuprofile, memprofile)() + if pedantic { moxvar.Pedantic = true } diff --git a/profile.go b/profile.go new file mode 100644 index 0000000..4441c5a --- /dev/null +++ b/profile.go @@ -0,0 +1,45 @@ +package main + +import ( + "log" + "os" + "runtime" + "runtime/pprof" +) + +func memprofile(mempath string) { + if mempath == "" { + return + } + + f, err := os.Create(mempath) + xcheckf(err, "creating memory profile") + defer func() { + if err := f.Close(); err != nil { + log.Printf("closing memory profile: %v", err) + } + }() + runtime.GC() // get up-to-date statistics + err = pprof.WriteHeapProfile(f) + xcheckf(err, "writing memory profile") +} + +func profile(cpupath, mempath string) func() { + if cpupath == "" { + return func() { + memprofile(mempath) + } + } + + f, err := os.Create(cpupath) + xcheckf(err, "creating CPU profile") + err = pprof.StartCPUProfile(f) + xcheckf(err, "start CPU profile") + return func() { + pprof.StopCPUProfile() + if err := f.Close(); err != nil { + log.Printf("closing cpu profile: %v", err) + } + memprofile(mempath) + } +}