for test-upgrade, import a (hopefully large) mbox file, checking for performance/memory consumption

in the future, it would be good to actually start a mox and read
mailboxes/messages...

.gitignore

@@ -23,6 +23,7 @@
 /testdata/smtpserverfuzz/data/
 /testdata/store/data/
 /testdata/train/
+/testdata/upgradetest.mbox.gz
 /testdata/integration/example-integration.zone
 /testdata/integration/tmp-pebble-ca.pem
 /cover.out

Makefile

@@ -19,8 +19,10 @@ test-race:
 	CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./...
 	go tool cover -html=cover.out -o cover.html
 
+# note: if testdata/upgradetest.mbox.gz exists, its messages will be imported
+# during tests. helpful for performance/resource consumption tests.
 test-upgrade:
-	./test-upgrade.sh
+	nice ./test-upgrade.sh
 
 check:
 	staticcheck ./...

README.md

@@ -180,6 +180,9 @@ and copy or move messages from one account to the other.
 Similarly, see the export functionality on the accounts web page and the "mox
 export maildir" and "mox export mbox" subcommands to export email.
 
+Importing large mailboxes may require a lot of memory (a limitation of the
+current database). Splitting up mailboxes in smaller parts would help.
+
 ## How can I help?
 
 Mox needs users and testing in real-life setups! So just give it a try, send

develop.txt

@@ -158,6 +158,39 @@ non-testing purposes. Unfortunately, this also makes it inconvenient to use for
 testing purposes.
 
 
+# Messages for testing
+
+For compatibility and preformance testing, it helps to have many messages,
+created a long time ago and recently, by different mail user agents. A helpful
+source is the Linux kernel mailing list. Archives are available as multiple git
+repositories (split due to size) at
+https://lore.kernel.org/lkml/_/text/mirror/.  The git repo's can be converted
+to compressed mbox files (about 800MB each) with:
+
+```
+# 0 is the first epoch (with over half a million messages), 12 is last
+# already-complete epoch at the time of writing (with a quarter million
+# messages). The archives are large, converting will take some time.
+for i in 0 12; do
+        git clone --mirror http://lore.kernel.org/lkml/$i lkml-$i.git
+        (cd lkml-$i.git && time ./tombox.sh | gzip >../lkml-$i.mbox.gz)
+done
+```
+
+With the following "tobmox.sh" script:
+
+```
+#!/bin/sh
+pre=''
+for rev in $(git rev-list master | reverse); do
+        printf "$pre"
+        echo "From sender@host  $(date '+%a %b %e %H:%M:%S %Y' -d @$(git show -s --format=%ct $rev))"
+        git show ${rev}:m | sed 's/^>*From />&/'
+        pre='\n'
+done
+```
+
+
 # Release proces
 
 - Gather feedback on recent changes.

test-upgrade.sh

@@ -1,10 +1,18 @@
 #!/bin/sh
 
+# note: If testdata/upgradetest.mbox.gz exists it will be imported it as part of
+# testing the upgrades. If this is a large mailbox, it will highlight performance
+# or resource consumption issues during upgrades.
+
 # todo: should we also test with mox.conf and domains.conf files? should "mox backup" and "mox gentestdata" add them, and "mox verifydata" use them?
 
 set -e
 # set -x
 
+# We'll allow max 256mb of memory during upgrades. We modify the softlimit when
+# importing the potentially large mbox file.
+ulimit -S -d 256000
+
 (rm -r testdata/upgrade 2>/dev/null || exit 0)
 mkdir testdata/upgrade
 cd testdata/upgrade
@@ -13,10 +21,11 @@ cd testdata/upgrade
 ../../mox gentestdata data
 ../../mox verifydata data
 rm -r data
+echo
 
 # For each historic release (i.e. all tagged versions) except the first few that
 # didn't have the gentestdata command, we generate a data directory for testing
-# and simulate upgrade to currently checked out version.
+# and simulate upgrading to the currently checked out version.
 # The awk command reverses the tags, so we try the previous release first since
 # it is the most likely to fail.
 tagsrev=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }')
@@ -29,6 +38,7 @@ for tag in $tagsrev; do
 	./$tag/mox gentestdata $tag/data
 	# Verify with current code.
 	../../mox verifydata $tag/data
+	echo
 	rm -r $tag/data
 done
 
@@ -40,14 +50,25 @@ for tag in $tags; do
 	if test "$first" = yes; then
 		echo "Starting with test data for $tag."
 		./$tag/mox gentestdata stepdata
+		echo
 		first=
 	else
+		# v0.0.5 got the ximport command
+		if test $tag = v0.0.5 -a -f ../upgradetest.mbox.gz; then
+			ulimit -S -d unlimited
+			echo 'Importing bulk data for upgrading.'
+			gunzip < ../upgradetest.mbox.gz | time ./$tag/mox ximport mbox ./stepdata/accounts/test0 upgradetest /dev/stdin
+			echo
+			ulimit -S -d 256000
+		fi
+
 		echo "Upgrade data to $tag."
-		./$tag/mox verifydata stepdata
+		time ./$tag/mox verifydata stepdata
+		echo
 	fi
 done
 echo "Testing final upgrade to current."
-../../mox verifydata stepdata
+time ../../mox verifydata stepdata
 rm -r stepdata
 rm */mox
 cd ../..