From 1ccc5d01770a66b2ee7d0e3ba8ddd4b96fa7248b Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Wed, 16 Aug 2023 14:36:17 +0200 Subject: [PATCH] fix message size in a message in gentestdata and work around the message in test-upgrade.sh. and add subcommand to open an account, triggering data upgrades. --- doc.go | 2 ++ gentestdata.go | 2 +- main.go | 25 +++++++++++++++++++++++++ store/account.go | 19 +++++++++++++------ test-upgrade.sh | 20 ++++++++++++++++---- verifydata.go | 8 +++++++- 6 files changed, 64 insertions(+), 12 deletions(-) diff --git a/doc.go b/doc.go index f895ca9..d7abf40 100644 --- a/doc.go +++ b/doc.go @@ -413,6 +413,8 @@ possibly making them potentially no longer readable by the previous version. usage: mox verifydata data-dir -fix fix fixable problems, such as moving away message files not referenced by their database + -skip-size-check + skip the check for message size # mox config test diff --git a/gentestdata.go b/gentestdata.go index 16b657f..4dbb1ba 100644 --- a/gentestdata.go +++ b/gentestdata.go @@ -356,7 +356,7 @@ Accounts: MailboxDestinedID: sent.ID, Flags: store.Flags{Seen: true, Junk: true}, Size: int64(len(prefix1) + len(msg1)), - MsgPrefix: []byte(prefix), + MsgPrefix: []byte(prefix1), } mf1 := tempfile() xcheckf(err, "creating temp file for delivery") diff --git a/main.go b/main.go index e4903a4..468844b 100644 --- a/main.go +++ b/main.go @@ -158,6 +158,7 @@ var commands = []struct { {"gentestdata", cmdGentestdata}, {"ximport maildir", cmdXImportMaildir}, {"ximport mbox", cmdXImportMbox}, + {"openaccounts", cmdOpenaccounts}, } var cmds []cmd @@ -2378,3 +2379,27 @@ func cmdMessageParse(c *cmd) { err = enc.Encode(part) xcheckf(err, "write") } + +func cmdOpenaccounts(c *cmd) { + c.unlisted = true + c.params = "datadir account ..." + c.help = `Open and close accounts, for triggering data upgrades, for tests. + +Opens database files directly, not going through a running mox instance. +` + + args := c.Parse() + if len(args) <= 1 { + c.Usage() + } + + dataDir := filepath.Clean(args[0]) + for _, accName := range args[1:] { + accDir := filepath.Join(dataDir, "accounts", accName) + log.Printf("opening account %s...", filepath.Join(accDir, accName)) + a, err := store.OpenAccountDB(accDir, accName) + xcheckf(err, "open account %s", accName) + err = a.Close() + xcheckf(err, "close account %s", accName) + } +} diff --git a/store/account.go b/store/account.go index 559c014..08cbe55 100644 --- a/store/account.go +++ b/store/account.go @@ -677,7 +677,6 @@ func OpenAccount(name string) (*Account, error) { if err != nil { return nil, err } - acc.nused++ openAccounts.names[name] = acc return acc, nil } @@ -685,13 +684,20 @@ func OpenAccount(name string) (*Account, error) { // openAccount opens an existing account, or creates it if it is missing. func openAccount(name string) (a *Account, rerr error) { dir := filepath.Join(mox.DataDirPath("accounts"), name) - dbpath := filepath.Join(dir, "index.db") + return OpenAccountDB(dir, name) +} + +// OpenAccountDB opens an account database file and returns an initialized account +// or error. Only exported for use by subcommands that verify the database file. +// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth. +func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { + dbpath := filepath.Join(accountDir, "index.db") // Create account if it doesn't exist yet. isNew := false if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) { isNew = true - os.MkdirAll(dir, 0770) + os.MkdirAll(accountDir, 0770) } db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) @@ -719,7 +725,7 @@ func openAccount(name string) (a *Account, rerr error) { return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error { if !mentioned { mentioned = true - xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", name)) + xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName)) } mc, err := mb.CalculateCounts(tx) if err != nil { @@ -736,10 +742,11 @@ func openAccount(name string) (a *Account, rerr error) { } return &Account{ - Name: name, - Dir: dir, + Name: accountName, + Dir: accountDir, DBPath: dbpath, DB: db, + nused: 1, }, nil } diff --git a/test-upgrade.sh b/test-upgrade.sh index a5b63b7..c82138f 100755 --- a/test-upgrade.sh +++ b/test-upgrade.sh @@ -22,6 +22,7 @@ cd testdata/upgrade # Check that we can upgrade what we currently generate. ../../mox gentestdata data ../../mox verifydata data +../../mox openaccounts data test0 test1 test2 rm -r data echo @@ -38,8 +39,13 @@ for tag in $tagsrev; do (CGO_ENABLED=0 GOBIN=$PWD/$tag go install github.com/mjl-/mox@$tag) # Generate with historic release. ./$tag/mox gentestdata $tag/data - # Verify with current code. - ../../mox verifydata $tag/data + # Verify with current code. v0.0.[45] had a message with wrong Size. We don't + # want to abort the upgrade check because of it. + if test $tag = v0.0.4 -o $tag = v0.0.5; then + ../../mox verifydata -skip-size-check $tag/data + else + ../../mox verifydata $tag/data + fi echo rm -r $tag/data done @@ -65,12 +71,18 @@ for tag in $tags; do fi echo "Upgrade data to $tag." - time ./$tag/mox verifydata stepdata + if test $tag = v0.0.4 -o $tag = v0.0.5; then + time ./$tag/mox verifydata stepdata + else + time ./$tag/mox verifydata -skip-size-check stepdata + time ./$tag/mox openaccounts stepdata test0 test1 test2 + fi echo fi done echo "Testing final upgrade to current." -time ../../mox verifydata stepdata +time ../../mox verifydata -skip-size-check stepdata +time ../../mox openaccounts stepdata test0 test1 test2 rm -r stepdata rm */mox cd ../.. diff --git a/verifydata.go b/verifydata.go index b1357b9..3479646 100644 --- a/verifydata.go +++ b/verifydata.go @@ -46,6 +46,12 @@ possibly making them potentially no longer readable by the previous version. ` var fix bool c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database") + + // To prevent aborting the upgrade test with v0.0.[45] that had a message with + // incorrect Size. + var skipSizeCheck bool + c.flag.BoolVar(&skipSizeCheck, "skip-size-check", false, "skip the check for message size") + args := c.Parse() if len(args) != 1 { c.Usage() @@ -140,7 +146,7 @@ possibly making them potentially no longer readable by the previous version. checkFile := func(dbpath, path string, prefixSize int, size int64) { st, err := os.Stat(path) checkf(err, path, "checking if file exists") - if err == nil && int64(prefixSize)+st.Size() != size { + if !skipSizeCheck && err == nil && int64(prefixSize)+st.Size() != size { filesize := st.Size() checkf(fmt.Errorf("%s: message size is %d, should be %d (length of MsgPrefix %d + file size %d), see \"mox fixmsgsize\"", path, size, int64(prefixSize)+st.Size(), prefixSize, filesize), dbpath, "checking message size") }