add subcommand "ximport", that is like "import" but directly access files in the datadir

so mox doesn't have to be running when you run it.
will be useful for testing in the near future.

this also moves cpuprof and memprof cli flags to top-level flag parsing, so all
commands can use them.

import.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"os"
 	"path/filepath"
 	"runtime/debug"
@@ -15,9 +17,11 @@ import (
 
 	"golang.org/x/exp/maps"
 
+	"github.com/mjl-/mox/config"
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/metrics"
 	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/store"
 )
 
@@ -89,6 +93,77 @@ func xcmdImport(mbox bool, args []string, c *cmd) {
 
 	ctl := xctl()
 	ctl.xwrite(ctlcmd)
+	xcmdImportCtl(ctl, account, mailbox, src)
+}
+
+func cmdXImportMaildir(c *cmd) {
+	c.unlisted = true
+	c.params = "accountdir mailboxname maildir"
+	c.help = `Import a maildir into an account by directly accessing the data directory.
+
+
+See "mox help import maildir" for details.
+`
+	xcmdXImport(false, c)
+}
+
+func cmdXImportMbox(c *cmd) {
+	c.unlisted = true
+	c.params = "accountdir mailboxname mbox"
+	c.help = `Import an mbox into an account by directly accessing the data directory.
+
+See "mox help import mbox" for details.
+`
+	xcmdXImport(true, c)
+}
+
+func xcmdXImport(mbox bool, c *cmd) {
+	args := c.Parse()
+	if len(args) != 3 {
+		c.Usage()
+	}
+
+	accountdir := args[0]
+	mailbox := args[1]
+	if strings.EqualFold(mailbox, "inbox") {
+		mailbox = "Inbox"
+	}
+	src := args[2]
+
+	var ctlcmd string
+	if mbox {
+		ctlcmd = "importmbox"
+	} else {
+		ctlcmd = "importmaildir"
+	}
+
+	account := filepath.Base(accountdir)
+
+	// Set up the mox config so the account can be opened.
+	if filepath.Base(filepath.Dir(accountdir)) != "accounts" {
+		log.Fatalf("accountdir must be of the form .../accounts/<name>")
+	}
+	var err error
+	mox.Conf.Static.DataDir, err = filepath.Abs(filepath.Dir(filepath.Dir(accountdir)))
+	xcheckf(err, "making absolute datadir")
+	mox.ConfigStaticPath = "fake.conf"
+	mox.Conf.DynamicLastCheck = time.Now().Add(time.Hour) // Silence errors about config file.
+	mox.Conf.Dynamic.Accounts = map[string]config.Account{
+		account: {},
+	}
+	switchDone := store.Switchboard()
+	defer close(switchDone)
+
+	xlog := mlog.New("import")
+	cconn, sconn := net.Pipe()
+	clientctl := ctl{conn: cconn, r: bufio.NewReader(cconn), log: xlog}
+	serverctl := ctl{cmd: ctlcmd, conn: sconn, r: bufio.NewReader(sconn), log: xlog}
+	go importctl(context.Background(), &serverctl, true)
+
+	xcmdImportCtl(&clientctl, account, mailbox, src)
+}
+
+func xcmdImportCtl(ctl *ctl, account, mailbox, src string) {
 	ctl.xwrite(account)
 	ctl.xwrite(mailbox)
 	ctl.xwrite(src)

junk.go

@@ -20,8 +20,6 @@ import (
 	"log"
 	mathrand "math/rand"
 	"os"
-	"runtime"
-	"runtime/pprof"
 	"sort"
 	"time"
 
@@ -33,7 +31,6 @@ import (
 
 type junkArgs struct {
 	params                        junk.Params
-	cpuprofile, memprofile        string
 	spamThreshold                 float64
 	trainRatio                    float64
 	seed                          bool
@@ -42,43 +39,6 @@ type junkArgs struct {
 	debug                         bool
 }
 
-func (a junkArgs) Memprofile() {
-	if a.memprofile == "" {
-		return
-	}
-
-	f, err := os.Create(a.memprofile)
-	xcheckf(err, "creating memory profile")
-	defer func() {
-		if err := f.Close(); err != nil {
-			log.Printf("closing memory profile: %v", err)
-		}
-	}()
-	runtime.GC() // get up-to-date statistics
-	err = pprof.WriteHeapProfile(f)
-	xcheckf(err, "writing memory profile")
-}
-
-func (a junkArgs) Profile() func() {
-	if a.cpuprofile == "" {
-		return func() {
-			a.Memprofile()
-		}
-	}
-
-	f, err := os.Create(a.cpuprofile)
-	xcheckf(err, "creating CPU profile")
-	err = pprof.StartCPUProfile(f)
-	xcheckf(err, "start CPU profile")
-	return func() {
-		pprof.StopCPUProfile()
-		if err := f.Close(); err != nil {
-			log.Printf("closing cpu profile: %v", err)
-		}
-		a.Memprofile()
-	}
-}
-
 func (a junkArgs) SetLogLevel() {
 	mox.Conf.Log[""] = mlog.LevelInfo
 	if a.debug {
@@ -104,8 +64,6 @@ func junkFlags(fs *flag.FlagSet) (a junkArgs) {
 	fs.StringVar(&a.databasePath, "dbpath", "filter.db", "database file for ham/spam words")
 	fs.StringVar(&a.bloomfilterPath, "bloompath", "filter.bloom", "bloom filter for ignoring unique strings")
 
-	fs.StringVar(&a.cpuprofile, "cpuprof", "", "store cpu profile to file")
-	fs.StringVar(&a.memprofile, "memprof", "", "store mem profile to file")
 	return
 }
 
@@ -132,7 +90,6 @@ func cmdJunkTrain(c *cmd) {
 	if len(args) != 2 {
 		c.Usage()
 	}
-	defer a.Profile()()
 	a.SetLogLevel()
 
 	f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath))
@@ -162,7 +119,6 @@ func cmdJunkCheck(c *cmd) {
 	if len(args) != 1 {
 		c.Usage()
 	}
-	defer a.Profile()()
 	a.SetLogLevel()
 
 	f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false))
@@ -187,7 +143,6 @@ func cmdJunkTest(c *cmd) {
 	if len(args) != 2 {
 		c.Usage()
 	}
-	defer a.Profile()()
 	a.SetLogLevel()
 
 	f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false))
@@ -244,7 +199,6 @@ messages are shuffled, with optional random seed.`
 	if len(args) != 2 {
 		c.Usage()
 	}
-	defer a.Profile()()
 	a.SetLogLevel()
 
 	f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath))
@@ -336,7 +290,6 @@ func cmdJunkPlay(c *cmd) {
 	if len(args) != 2 {
 		c.Usage()
 	}
-	defer a.Profile()()
 	a.SetLogLevel()
 
 	f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath))

main.go

@@ -152,6 +152,8 @@ var commands = []struct {
 	{"updates serve", cmdUpdatesServe},
 	{"updates verify", cmdUpdatesVerify},
 	{"gentestdata", cmdGentestdata},
+	{"ximport maildir", cmdXImportMaildir},
+	{"ximport mbox", cmdXImportMbox},
 }
 
 var cmds []cmd
@@ -391,12 +393,19 @@ func main() {
 	flag.StringVar(&loglevel, "loglevel", "", "if non-empty, this log level is set early in startup")
 	flag.BoolVar(&pedantic, "pedantic", false, "protocol violations result in errors instead of accepting/working around them")
 
+	var cpuprofile, memprofile string
+	flag.StringVar(&cpuprofile, "cpuprof", "", "store cpu profile to file")
+	flag.StringVar(&memprofile, "memprof", "", "store mem profile to file")
+
 	flag.Usage = func() { usage(cmds, false) }
 	flag.Parse()
 	args := flag.Args()
 	if len(args) == 0 {
 		usage(cmds, false)
 	}
+
+	defer profile(cpuprofile, memprofile)()
+
 	if pedantic {
 		moxvar.Pedantic = true
 	}

profile.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"log"
+	"os"
+	"runtime"
+	"runtime/pprof"
+)
+
+func memprofile(mempath string) {
+	if mempath == "" {
+		return
+	}
+
+	f, err := os.Create(mempath)
+	xcheckf(err, "creating memory profile")
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Printf("closing memory profile: %v", err)
+		}
+	}()
+	runtime.GC() // get up-to-date statistics
+	err = pprof.WriteHeapProfile(f)
+	xcheckf(err, "writing memory profile")
+}
+
+func profile(cpupath, mempath string) func() {
+	if cpupath == "" {
+		return func() {
+			memprofile(mempath)
+		}
+	}
+
+	f, err := os.Create(cpupath)
+	xcheckf(err, "creating CPU profile")
+	err = pprof.StartCPUProfile(f)
+	xcheckf(err, "start CPU profile")
+	return func() {
+		pprof.StopCPUProfile()
+		if err := f.Close(); err != nil {
+			log.Printf("closing cpu profile: %v", err)
+		}
+		memprofile(mempath)
+	}
+}