diff --git a/import.go b/import.go index df2f978..2905da6 100644 --- a/import.go +++ b/import.go @@ -1,12 +1,14 @@ package main import ( + "bufio" "context" "encoding/json" "errors" "fmt" "io" "log" + "net" "os" "path/filepath" "runtime/debug" @@ -15,9 +17,11 @@ import ( "golang.org/x/exp/maps" + "github.com/mjl-/mox/config" "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) @@ -89,6 +93,77 @@ func xcmdImport(mbox bool, args []string, c *cmd) { ctl := xctl() ctl.xwrite(ctlcmd) + xcmdImportCtl(ctl, account, mailbox, src) +} + +func cmdXImportMaildir(c *cmd) { + c.unlisted = true + c.params = "accountdir mailboxname maildir" + c.help = `Import a maildir into an account by directly accessing the data directory. + + +See "mox help import maildir" for details. +` + xcmdXImport(false, c) +} + +func cmdXImportMbox(c *cmd) { + c.unlisted = true + c.params = "accountdir mailboxname mbox" + c.help = `Import an mbox into an account by directly accessing the data directory. + +See "mox help import mbox" for details. +` + xcmdXImport(true, c) +} + +func xcmdXImport(mbox bool, c *cmd) { + args := c.Parse() + if len(args) != 3 { + c.Usage() + } + + accountdir := args[0] + mailbox := args[1] + if strings.EqualFold(mailbox, "inbox") { + mailbox = "Inbox" + } + src := args[2] + + var ctlcmd string + if mbox { + ctlcmd = "importmbox" + } else { + ctlcmd = "importmaildir" + } + + account := filepath.Base(accountdir) + + // Set up the mox config so the account can be opened. + if filepath.Base(filepath.Dir(accountdir)) != "accounts" { + log.Fatalf("accountdir must be of the form .../accounts/") + } + var err error + mox.Conf.Static.DataDir, err = filepath.Abs(filepath.Dir(filepath.Dir(accountdir))) + xcheckf(err, "making absolute datadir") + mox.ConfigStaticPath = "fake.conf" + mox.Conf.DynamicLastCheck = time.Now().Add(time.Hour) // Silence errors about config file. + mox.Conf.Dynamic.Accounts = map[string]config.Account{ + account: {}, + } + switchDone := store.Switchboard() + defer close(switchDone) + + xlog := mlog.New("import") + cconn, sconn := net.Pipe() + clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: xlog} + serverctl := ctl{cmd: ctlcmd, conn: sconn, r: bufio.NewReader(sconn), log: xlog} + go importctl(context.Background(), &serverctl, true) + + xcmdImportCtl(&clientctl, account, mailbox, src) +} + +func xcmdImportCtl(ctl *ctl, account, mailbox, src string) { ctl.xwrite(account) ctl.xwrite(mailbox) ctl.xwrite(src) diff --git a/junk.go b/junk.go index 277fbc4..83ba273 100644 --- a/junk.go +++ b/junk.go @@ -20,8 +20,6 @@ import ( "log" mathrand "math/rand" "os" - "runtime" - "runtime/pprof" "sort" "time" @@ -33,7 +31,6 @@ import ( type junkArgs struct { params junk.Params - cpuprofile, memprofile string spamThreshold float64 trainRatio float64 seed bool @@ -42,43 +39,6 @@ type junkArgs struct { debug bool } -func (a junkArgs) Memprofile() { - if a.memprofile == "" { - return - } - - f, err := os.Create(a.memprofile) - xcheckf(err, "creating memory profile") - defer func() { - if err := f.Close(); err != nil { - log.Printf("closing memory profile: %v", err) - } - }() - runtime.GC() // get up-to-date statistics - err = pprof.WriteHeapProfile(f) - xcheckf(err, "writing memory profile") -} - -func (a junkArgs) Profile() func() { - if a.cpuprofile == "" { - return func() { - a.Memprofile() - } - } - - f, err := os.Create(a.cpuprofile) - xcheckf(err, "creating CPU profile") - err = pprof.StartCPUProfile(f) - xcheckf(err, "start CPU profile") - return func() { - pprof.StopCPUProfile() - if err := f.Close(); err != nil { - log.Printf("closing cpu profile: %v", err) - } - a.Memprofile() - } -} - func (a junkArgs) SetLogLevel() { mox.Conf.Log[""] = mlog.LevelInfo if a.debug { @@ -104,8 +64,6 @@ func junkFlags(fs *flag.FlagSet) (a junkArgs) { fs.StringVar(&a.databasePath, "dbpath", "filter.db", "database file for ham/spam words") fs.StringVar(&a.bloomfilterPath, "bloompath", "filter.bloom", "bloom filter for ignoring unique strings") - fs.StringVar(&a.cpuprofile, "cpuprof", "", "store cpu profile to file") - fs.StringVar(&a.memprofile, "memprof", "", "store mem profile to file") return } @@ -132,7 +90,6 @@ func cmdJunkTrain(c *cmd) { if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath)) @@ -162,7 +119,6 @@ func cmdJunkCheck(c *cmd) { if len(args) != 1 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false)) @@ -187,7 +143,6 @@ func cmdJunkTest(c *cmd) { if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false)) @@ -244,7 +199,6 @@ messages are shuffled, with optional random seed.` if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath)) @@ -336,7 +290,6 @@ func cmdJunkPlay(c *cmd) { if len(args) != 2 { c.Usage() } - defer a.Profile()() a.SetLogLevel() f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath)) diff --git a/main.go b/main.go index 543268a..2c659ae 100644 --- a/main.go +++ b/main.go @@ -152,6 +152,8 @@ var commands = []struct { {"updates serve", cmdUpdatesServe}, {"updates verify", cmdUpdatesVerify}, {"gentestdata", cmdGentestdata}, + {"ximport maildir", cmdXImportMaildir}, + {"ximport mbox", cmdXImportMbox}, } var cmds []cmd @@ -391,12 +393,19 @@ func main() { flag.StringVar(&loglevel, "loglevel", "", "if non-empty, this log level is set early in startup") flag.BoolVar(&pedantic, "pedantic", false, "protocol violations result in errors instead of accepting/working around them") + var cpuprofile, memprofile string + flag.StringVar(&cpuprofile, "cpuprof", "", "store cpu profile to file") + flag.StringVar(&memprofile, "memprof", "", "store mem profile to file") + flag.Usage = func() { usage(cmds, false) } flag.Parse() args := flag.Args() if len(args) == 0 { usage(cmds, false) } + + defer profile(cpuprofile, memprofile)() + if pedantic { moxvar.Pedantic = true } diff --git a/profile.go b/profile.go new file mode 100644 index 0000000..4441c5a --- /dev/null +++ b/profile.go @@ -0,0 +1,45 @@ +package main + +import ( + "log" + "os" + "runtime" + "runtime/pprof" +) + +func memprofile(mempath string) { + if mempath == "" { + return + } + + f, err := os.Create(mempath) + xcheckf(err, "creating memory profile") + defer func() { + if err := f.Close(); err != nil { + log.Printf("closing memory profile: %v", err) + } + }() + runtime.GC() // get up-to-date statistics + err = pprof.WriteHeapProfile(f) + xcheckf(err, "writing memory profile") +} + +func profile(cpupath, mempath string) func() { + if cpupath == "" { + return func() { + memprofile(mempath) + } + } + + f, err := os.Create(cpupath) + xcheckf(err, "creating CPU profile") + err = pprof.StartCPUProfile(f) + xcheckf(err, "start CPU profile") + return func() { + pprof.StopCPUProfile() + if err := f.Close(); err != nil { + log.Printf("closing cpu profile: %v", err) + } + memprofile(mempath) + } +}