diff --git a/.gitignore b/.gitignore index 0eab6c1..8c0cf05 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ /testdata/smtpserverfuzz/data/ /testdata/store/data/ /testdata/train/ +/testdata/upgradetest.mbox.gz /testdata/integration/example-integration.zone /testdata/integration/tmp-pebble-ca.pem /cover.out diff --git a/Makefile b/Makefile index 1611bc9..120f530 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,10 @@ test-race: CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./... go tool cover -html=cover.out -o cover.html +# note: if testdata/upgradetest.mbox.gz exists, its messages will be imported +# during tests. helpful for performance/resource consumption tests. test-upgrade: - ./test-upgrade.sh + nice ./test-upgrade.sh check: staticcheck ./... diff --git a/README.md b/README.md index 2e187df..41e91c4 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,9 @@ and copy or move messages from one account to the other. Similarly, see the export functionality on the accounts web page and the "mox export maildir" and "mox export mbox" subcommands to export email. +Importing large mailboxes may require a lot of memory (a limitation of the +current database). Splitting up mailboxes in smaller parts would help. + ## How can I help? Mox needs users and testing in real-life setups! So just give it a try, send diff --git a/develop.txt b/develop.txt index d91040f..1bebafb 100644 --- a/develop.txt +++ b/develop.txt @@ -158,6 +158,39 @@ non-testing purposes. Unfortunately, this also makes it inconvenient to use for testing purposes. +# Messages for testing + +For compatibility and preformance testing, it helps to have many messages, +created a long time ago and recently, by different mail user agents. A helpful +source is the Linux kernel mailing list. Archives are available as multiple git +repositories (split due to size) at +https://lore.kernel.org/lkml/_/text/mirror/. The git repo's can be converted +to compressed mbox files (about 800MB each) with: + +``` +# 0 is the first epoch (with over half a million messages), 12 is last +# already-complete epoch at the time of writing (with a quarter million +# messages). The archives are large, converting will take some time. +for i in 0 12; do + git clone --mirror http://lore.kernel.org/lkml/$i lkml-$i.git + (cd lkml-$i.git && time ./tombox.sh | gzip >../lkml-$i.mbox.gz) +done +``` + +With the following "tobmox.sh" script: + +``` +#!/bin/sh +pre='' +for rev in $(git rev-list master | reverse); do + printf "$pre" + echo "From sender@host $(date '+%a %b %e %H:%M:%S %Y' -d @$(git show -s --format=%ct $rev))" + git show ${rev}:m | sed 's/^>*From />&/' + pre='\n' +done +``` + + # Release proces - Gather feedback on recent changes. diff --git a/test-upgrade.sh b/test-upgrade.sh index dbd7565..025852a 100755 --- a/test-upgrade.sh +++ b/test-upgrade.sh @@ -1,10 +1,18 @@ #!/bin/sh +# note: If testdata/upgradetest.mbox.gz exists it will be imported it as part of +# testing the upgrades. If this is a large mailbox, it will highlight performance +# or resource consumption issues during upgrades. + # todo: should we also test with mox.conf and domains.conf files? should "mox backup" and "mox gentestdata" add them, and "mox verifydata" use them? set -e # set -x +# We'll allow max 256mb of memory during upgrades. We modify the softlimit when +# importing the potentially large mbox file. +ulimit -S -d 256000 + (rm -r testdata/upgrade 2>/dev/null || exit 0) mkdir testdata/upgrade cd testdata/upgrade @@ -13,10 +21,11 @@ cd testdata/upgrade ../../mox gentestdata data ../../mox verifydata data rm -r data +echo # For each historic release (i.e. all tagged versions) except the first few that # didn't have the gentestdata command, we generate a data directory for testing -# and simulate upgrade to currently checked out version. +# and simulate upgrading to the currently checked out version. # The awk command reverses the tags, so we try the previous release first since # it is the most likely to fail. tagsrev=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') @@ -29,6 +38,7 @@ for tag in $tagsrev; do ./$tag/mox gentestdata $tag/data # Verify with current code. ../../mox verifydata $tag/data + echo rm -r $tag/data done @@ -40,14 +50,25 @@ for tag in $tags; do if test "$first" = yes; then echo "Starting with test data for $tag." ./$tag/mox gentestdata stepdata + echo first= else + # v0.0.5 got the ximport command + if test $tag = v0.0.5 -a -f ../upgradetest.mbox.gz; then + ulimit -S -d unlimited + echo 'Importing bulk data for upgrading.' + gunzip < ../upgradetest.mbox.gz | time ./$tag/mox ximport mbox ./stepdata/accounts/test0 upgradetest /dev/stdin + echo + ulimit -S -d 256000 + fi + echo "Upgrade data to $tag." - ./$tag/mox verifydata stepdata + time ./$tag/mox verifydata stepdata + echo fi done echo "Testing final upgrade to current." -../../mox verifydata stepdata +time ../../mox verifydata stepdata rm -r stepdata rm */mox cd ../..