mox!

.dockerignore

@@ -0,0 +1,8 @@
+/mox
+/testdata/
+/node_modules/
+/local/
+/rfc/
+/cover.*
+/.go/
+/tmp/

.gitignore

@@ -0,0 +1,22 @@
+/mox
+/rfc/[0-9][0-9]*
+/local/
+/testdata/check/
+/testdata/empty/
+/testdata/imap/data/
+/testdata/imaptest/data/
+/testdata/integration/run
+/testdata/junk/*.bloom
+/testdata/junk/*.db
+/testdata/queue/data/
+/testdata/sent/
+/testdata/smtp/data/
+/testdata/smtp/datajunk/
+/testdata/store/data/
+/testdata/train/
+/cover.out
+/cover.html
+/.go/
+/node_modules/
+/package.json
+/package-lock.json

.jshintrc

@@ -0,0 +1,12 @@
+{
+	"esversion": 9,
+	"asi": true,
+	"strict": "implied",
+	"globals": {
+		"window": true,
+		"console": true,
+		"document": true,
+		"Node": true,
+		"api": true
+	}
+}

Dockerfile

@@ -0,0 +1,11 @@
+FROM golang:1-alpine AS build
+WORKDIR /build
+RUN apk add make
+COPY . .
+env GOPROXY=off
+RUN make build
+
+FROM alpine:3.17
+WORKDIR /mox
+COPY --from=build /build/mox /mox/mox
+CMD ["/mox/mox", "serve"]

Dockerfile.imaptest

@@ -0,0 +1,7 @@
+FROM alpine:3.17
+
+RUN apk update && apk add wget build-base
+WORKDIR /src
+RUN wget http://dovecot.org/nightly/dovecot-latest.tar.gz && tar -zxvf dovecot-latest.tar.gz && cd dovecot-0.0.0-* && ./configure && make install && cd ..
+RUN wget http://dovecot.org/nightly/imaptest/imaptest-latest.tar.gz && tar -zxvf imaptest-latest.tar.gz && cd dovecot-0.0-imaptest-0.0.0-* && ./configure --with-dovecot=$(ls -d ../dovecot-0.0.0-*) && make install
+ENTRYPOINT /usr/local/bin/imaptest

LICENSE.MIT

@@ -0,0 +1,7 @@
+Copyright 2021 Mechiel Lukkien <mechiel@ueber.net>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.MPLv2.0

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

Makefile

@@ -0,0 +1,76 @@
+default: build
+
+build:
+	# build early to catch syntax errors
+	CGO_ENABLED=0 go build
+	CGO_ENABLED=0 go vet -tags integration ./...
+	./gendoc.sh
+	(cd http && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none Admin) >http/adminapi.json
+	(cd http && CGO_ENABLED=0 go run ../vendor/github.com/mjl-/sherpadoc/cmd/sherpadoc/*.go -adjust-function-names none Account) >http/accountapi.json
+	# build again, files above are embedded
+	CGO_ENABLED=0 go build
+
+test:
+	CGO_ENABLED=0 go test -shuffle=on -coverprofile cover.out ./...
+	go tool cover -html=cover.out -o cover.html
+
+test-race:
+	CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./...
+	go tool cover -html=cover.out -o cover.html
+
+check:
+	staticcheck ./...
+	staticcheck -tags integration
+
+# having "err" shadowed is common, best to not have others
+check-shadow:
+	go vet -vettool=$$(which shadow) ./...
+
+fuzz:
+	go test -fuzz FuzzParseSignature -fuzztime 5m ./dkim
+	go test -fuzz FuzzParseRecord -fuzztime 5m ./dkim
+	go test -fuzz . -fuzztime 5m ./dmarc
+	go test -fuzz . -fuzztime 5m ./dmarcrpt
+	go test -fuzz . -parallel 1 -fuzztime 5m ./imapserver
+	go test -fuzz . -parallel 1 -fuzztime 5m ./junk
+	go test -fuzz FuzzParseRecord -fuzztime 5m ./mtasts
+	go test -fuzz FuzzParsePolicy -fuzztime 5m ./mtasts
+	go test -fuzz . -parallel 1 -fuzztime 5m ./smtpserver
+	go test -fuzz . -fuzztime 5m ./spf
+	go test -fuzz FuzzParseRecord -fuzztime 5m ./tlsrpt
+	go test -fuzz FuzzParseMessage -fuzztime 5m ./tlsrpt
+
+integration-build:
+	docker-compose -f docker-compose-integration.yml build --no-cache moxmail
+
+integration-start:
+	-MOX_UID=$$(id -u) MOX_GID=$$(id -g) docker-compose -f docker-compose-integration.yml run moxmail /bin/bash
+	MOX_UID= MOX_GID= docker-compose -f docker-compose-integration.yml down
+
+# run from within "make integration-start"
+integration-test:
+	CGO_ENABLED=0 go test -tags integration
+	go tool cover -html=cover.out -o cover.html
+
+imaptest-build:
+	-MOX_UID=$$(id -u) MOX_GID=$$(id -g) docker-compose -f docker-compose-imaptest.yml build --no-cache mox
+
+imaptest-run:
+	-rm -r testdata/imaptest/data
+	mkdir testdata/imaptest/data
+	MOX_UID=$$(id -u) MOX_GID=$$(id -g) docker-compose -f docker-compose-imaptest.yml run --entrypoint /usr/local/bin/imaptest imaptest host=mox port=1143 user=mjl@mox.example pass=testtest mbox=imaptest.mbox
+	MOX_UID= MOX_GID= docker-compose -f docker-compose-imaptest.yml down
+
+fmt:
+	go fmt ./...
+	gofmt -w -s *.go */*.go
+
+jswatch:
+	inotifywait -m -e close_write http/admin.html http/account.html | xargs -n2 sh -c 'echo changed; ./checkhtmljs http/admin.html http/account.html'
+
+jsinstall:
+	-mkdir -p node_modules/.bin
+	npm install jshint@2.13.2
+
+docker:
+	docker build -t mox:latest .

README.md

@@ -0,0 +1,181 @@
+Mox - modern full-featured open source secure mail server for low-maintenance self-hosted email
+
+See Quickstart below to get started.
+
+Mox features:
+
+- Quick and easy to maintain mail server for your own domain through quickstart.
+- SMTP for receiving and submitting email.
+- IMAP4 for giving email clients access to email.
+- Automatic TLS with ACME, for use with Let's Encrypt and other CA's.
+- SPF, verifying that a remote host is allowed to sent email for a domain.
+- DKIM, verifying that a message is signed by the claimed sender domain,
+  and for signing emails sent by mox for others to verify.
+- DMARC, for enforcing SPF/DKIM policies set by domains. Incoming DMARC
+  aggregate reports are analyzed.
+- Reputation tracking, learning (per user) host- and domain-based reputation from
+  (Non-)Junk/Non-Junk email.
+- Bayesian spam filtering that learns (per user) from (Non-)Junk email.
+- Greylisting of servers with no/low reputation and questionable email content.
+  Temporarily refused emails are available over IMAP in a special mailbox for a
+  short period, helping with misclassified legimate synchronous
+  signup/login/transactional emails.
+- Internationalized email, with unicode names in domains and usernames
+  ("localparts").
+- TLSRPT, parsing reports about TLS usage and issues.
+- MTA-STS, for ensuring TLS is used whenever it is required. Both serving of
+  policies, and tracking and applying policies of remote servers.
+- Web admin interface that helps you set up your domains and accounts
+  (instructions to create DNS records, configure
+  SPF/DKIM/DMARC/TLSRPT/MTA-STS), for status information, managing
+  accounts/domains, and modifying the configuration file.
+- Autodiscovery (with SRV records, Microsoft-style and Thunderbird-style) for
+  easy account setup (though not many clients support it).
+- Prometheus metrics and structured logging for operational insight.
+
+Not supported (but perhaps in the future):
+
+- Webmail
+- Functioning as SMTP relay
+- HTTP-based API for sending messages and receiving delivery feedback
+- Forwarding (to an external address)
+- Autoresponders
+- POP3
+- Delivery to (unix) OS system users
+- Sieve for filtering
+- PGP or S/MIME
+- Mailing list manager
+- Calendaring
+- Support for pluggable delivery mechanisms.
+
+Mox has automated tests, including for interoperability with Postfix for SMTP.
+
+Mox is manually tested with email clients: Mozilla Thunderbird, mutt, iOS Mail,
+macOS Mail, Android Mail, Microsoft Outlook.
+
+Mox is also manually tested to interoperate with popular cloud providers:
+gmail.com, outlook.com, yahoo.com, proton.me.
+
+Mox is implemented in Go, a modern safe programming language, and has a focus on
+security.
+
+Mox is available under the MIT-license.
+Mox includes the Public Suffix List by Mozilla, under Mozilla Public License, v. 2.0.
+
+Mox was created by Mechiel Lukkien, mechiel@ueber.net.
+
+
+# Download
+
+You can easily (cross) compile mox if you have a Go toolchain installed:
+
+	go install github.com/mjl-/mox@latest
+
+Or you can download binaries from:
+
+	https://beta.gobuilds.org/github.com/mjl-/mox
+
+
+# Quickstart
+
+The easiest way to get started with serving email for your domain is to get a
+vm/machine dedicated to serving email named <host>.<domain>, login as an admin
+user, e.g. /home/service, download mox, and generate a configuration for your
+desired email address at your domain:
+
+	./mox quickstart you@example.com
+
+This creates an accounts, generates a password and configuration files, prints
+the DNS records you need to manually add for your domain and prints commands to
+set permissions and install as a service.
+
+If you already have email configured for your domain, or if you are already
+sending email for your domain from other machines/services, you should modify
+the suggested configuration and/or DNS records.
+
+A dedicated machine is convenient because modern email requires HTTPS.  You can
+combine mox with an existing webserver, but it requires more configuration.
+
+After starting, you can access the admin web interface on internal IPs.
+
+
+# FAQ - Frequently Asked Questions
+
+- Why a new mail server implementation?
+
+Mox aims to make "running a mail server" easy and nearly effortless. Excellent
+quality mail server software exists, but getting a working setup typically
+requires you configure half a dozen services (SMTP, IMAP, SPF/DKIM/DMARC, spam
+filtering). That seems to lead to people no longer running their own mail
+servers, instead switching to one of the few centralized email providers. SMTP
+is long-time distributed messaging protocol. To keep it distributed, people
+need to run their own mail server. Mox aims to make that easy.
+
+- Where is the documentation?
+
+Run "mox" without arguments to list its subcommands and usage, run "mox help
+<subcommand>" for more details. See all commands and help text at:
+
+	https://pkg.go.dev/github.com/mjl-/mox/
+
+The example configuration files annotated with comments can be helpful too.
+They are printed by "mox config describe-static" and "mox config
+describe-dynamic", and can be viewed at:
+
+	https://pkg.go.dev/github.com/mjl-/mox/config/
+
+Mox is still in early stages, and documentation is still limited. Please create
+an issue describing what is unclear or confusing, and we'll try to improve the
+documentation.
+
+- How do I import/export email?
+
+Use the "mox import maildir" or "mox import mbox" subcommands. You could also
+use your IMAP email client, add your mox account, and copy or move messages
+from one account to the other.
+
+Similarly, see the "mox export maildir" and "mox export mbox" subcommands to
+export email.
+
+- How can I help?
+
+Mox needs users and testing in real-life setups! So just give it a try, send
+and receive emails through it with your favourite email clients, and file an
+issue if you encounter a problem or would like to see a feature/functionality
+implemented.
+
+Instead of switching your email for your domain over to mox, you could simply
+configure mox for a subdomain, e.g. <you>@moxtest.<yourdomain>.
+
+If you have experience with how the email protocols are used in the wild, e.g.
+compatibility issues, limitations, anti-spam measures, specification
+violations, that would be interesting to hear about.
+
+Pull requests for bug fixes and new code are welcome too. If the changes are
+large, it helps to start a discussion (create a ticket) before doing all the
+work.
+
+- How do I change my password?
+
+Regular users (doing IMAP/SMTP with authentication) can change their password
+at the account page, e.g. http://127.0.0.1/account/. Or you can set a password
+with "mox setaccountpassword".
+
+The admin password can be changed with "mox setadminpassword".
+
+- How do I configure a second mox instance as a backup MX?
+
+Unfortunately, mox does not yet provide an option for that. Mox does spam
+filtering based on reputation of received messages. It will take a good amount
+of work to share that information with a backup MX. Without that information,
+spammer could use a backup MX to get their spam accepted. Until mox has a
+proper solution, you can simply run a single SMTP server.
+
+- How secure is mox?
+
+Security is high on the priorit list for mox. Mox is young, so don't expect no
+bugs at all. Mox does have automated tests for some security aspects, e.g. for
+login, and uses fuzzing. Mox is written in Go, so some classes of bugs such as
+buffer mishandling do not typically result in privilege escalation.  Of course
+logic bugs will still exist. If you find any security issues, please email them
+to mechiel@ueber.net.

autotls/autotls.go

@@ -0,0 +1,279 @@
+// Package autotls automatically configures TLS (for SMTP, IMAP, HTTP) by
+// requesting certificates with ACME, typically from Let's Encrypt.
+package autotls
+
+// We only do tls-alpn-01. For http-01, we would have to start another
+// listener. For DNS we would need a third party tool with an API that can make
+// the DNS changes, as we don't want to link in dozens of bespoke API's for DNS
+// record manipulation into mox. We can do http-01 relatively easily. It could
+// be useful to not depend on a single mechanism.
+
+import (
+	"bytes"
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"golang.org/x/crypto/acme"
+	"golang.org/x/crypto/acme/autocert"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxvar"
+)
+
+var xlog = mlog.New("autotls")
+
+var (
+	metricCertput = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "mox_autotls_certput_total",
+			Help: "Number of certificate store puts.",
+		},
+	)
+)
+
+// Manager is in charge of a single ACME identity, and automatically requests
+// certificates for allowlisted hosts.
+type Manager struct {
+	ACMETLSConfig *tls.Config // For serving HTTPS on port 443, which is required for certificate requests to succeed.
+	TLSConfig     *tls.Config // For all TLS servers not used for validating ACME requests. Like SMTP and HTTPS on ports other than 443.
+	Manager       *autocert.Manager
+
+	shutdown <-chan struct{}
+
+	sync.Mutex
+	hosts map[dns.Domain]struct{}
+}
+
+// Load returns an initialized autotls manager for "name" (used for the ACME key
+// file and requested certs and their keys). All files are stored within acmeDir.
+// contactEmail must be a valid email address to which notifications about ACME can
+// be sent. directoryURL is the ACME starting point. When shutdown is closed, no
+// new TLS connections can be created.
+func Load(name, acmeDir, contactEmail, directoryURL string, shutdown <-chan struct{}) (*Manager, error) {
+	if directoryURL == "" {
+		return nil, fmt.Errorf("empty ACME directory URL")
+	}
+	if contactEmail == "" {
+		return nil, fmt.Errorf("empty contact email")
+	}
+
+	// Load identity key if it exists. Otherwise, create a new key.
+	p := filepath.Join(acmeDir + "/" + name + ".key")
+	var key crypto.Signer
+	f, err := os.Open(p)
+	if f != nil {
+		defer f.Close()
+	}
+	if err != nil && os.IsNotExist(err) {
+		key, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+		if err != nil {
+			return nil, fmt.Errorf("generating ecdsa identity key: %s", err)
+		}
+		der, err := x509.MarshalPKCS8PrivateKey(key)
+		if err != nil {
+			return nil, fmt.Errorf("marshal identity key: %s", err)
+		}
+		block := &pem.Block{
+			Type: "PRIVATE KEY",
+			Headers: map[string]string{
+				"Note": fmt.Sprintf("PEM PKCS8 ECDSA private key generated for ACME provider %s by mox", name),
+			},
+			Bytes: der,
+		}
+		b := &bytes.Buffer{}
+		if err := pem.Encode(b, block); err != nil {
+			return nil, fmt.Errorf("pem encode: %s", err)
+		} else if err := os.WriteFile(p, b.Bytes(), 0660); err != nil {
+			return nil, fmt.Errorf("writing identity key: %s", err)
+		}
+	} else if err != nil {
+		return nil, fmt.Errorf("open identity key file: %s", err)
+	} else {
+		var privKey any
+		if buf, err := io.ReadAll(f); err != nil {
+			return nil, fmt.Errorf("reading identity key: %s", err)
+		} else if p, _ := pem.Decode(buf); p == nil {
+			return nil, fmt.Errorf("no pem data")
+		} else if p.Type != "PRIVATE KEY" {
+			return nil, fmt.Errorf("got PEM block %q, expected \"PRIVATE KEY\"", p.Type)
+		} else if privKey, err = x509.ParsePKCS8PrivateKey(p.Bytes); err != nil {
+			return nil, fmt.Errorf("parsing PKCS8 private key: %s", err)
+		}
+		switch k := privKey.(type) {
+		case *ecdsa.PrivateKey:
+			key = k
+		case *rsa.PrivateKey:
+			key = k
+		default:
+			return nil, fmt.Errorf("unsupported private key type %T", key)
+		}
+	}
+
+	m := &autocert.Manager{
+		Cache:  dirCache(acmeDir + "/keycerts/" + name),
+		Prompt: autocert.AcceptTOS,
+		Email:  contactEmail,
+		Client: &acme.Client{
+			DirectoryURL: directoryURL,
+			Key:          key,
+			UserAgent:    "mox/" + moxvar.Version,
+		},
+		// HostPolicy set below.
+	}
+
+	loggingGetCertificate := func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		log := xlog.WithContext(hello.Context())
+
+		// Handle missing SNI to prevent logging an error below.
+		// At startup, during config initialization, we already adjust the tls config to
+		// inject the listener hostname if there isn't one in the TLS client hello. This is
+		// common for SMTP STARTTLS connections, which often do not care about the
+		// validation of the certificate.
+		if hello.ServerName == "" {
+			log.Debug("tls request without sni servername, rejecting")
+			return nil, fmt.Errorf("sni server name required")
+		}
+
+		cert, err := m.GetCertificate(hello)
+		if err != nil {
+			if errors.Is(err, errHostNotAllowed) {
+				log.Debugx("requesting certificate", err, mlog.Field("host", hello.ServerName))
+			} else {
+				log.Errorx("requesting certificate", err, mlog.Field("host", hello.ServerName))
+			}
+		}
+		return cert, err
+	}
+
+	acmeTLSConfig := *m.TLSConfig()
+	acmeTLSConfig.GetCertificate = loggingGetCertificate
+
+	tlsConfig := tls.Config{
+		GetCertificate: loggingGetCertificate,
+	}
+
+	a := &Manager{
+		ACMETLSConfig: &acmeTLSConfig,
+		TLSConfig:     &tlsConfig,
+		Manager:       m,
+		shutdown:      shutdown,
+		hosts:         map[dns.Domain]struct{}{},
+	}
+	m.HostPolicy = a.HostPolicy
+	return a, nil
+}
+
+// AllowHostname adds hostname for use with ACME.
+func (m *Manager) AllowHostname(hostname dns.Domain) {
+	m.Lock()
+	defer m.Unlock()
+	xlog.Debug("autotls add hostname", mlog.Field("hostname", hostname))
+	m.hosts[hostname] = struct{}{}
+}
+
+// Hostnames returns the allowed host names for use with ACME.
+func (m *Manager) Hostnames() []dns.Domain {
+	m.Lock()
+	defer m.Unlock()
+	var l []dns.Domain
+	for h := range m.hosts {
+		l = append(l, h)
+	}
+	return l
+}
+
+var errHostNotAllowed = errors.New("autotls: host not in allowlist")
+
+// HostPolicy decides if a host is allowed for use with ACME, i.e. whether a
+// certificate will be returned if present and/or will be requested if not yet
+// present. Only hosts added with AllowHostname are allowed. During shutdown, no
+// new connections are allowed.
+func (m *Manager) HostPolicy(ctx context.Context, host string) (rerr error) {
+	log := xlog.WithContext(ctx)
+	defer func() {
+		log.WithContext(ctx).Debugx("autotls hostpolicy result", rerr, mlog.Field("host", host))
+	}()
+
+	// Don't request new TLS certs when we are shutting down.
+	select {
+	case <-m.shutdown:
+		return fmt.Errorf("shutting down")
+	default:
+	}
+
+	d, err := dns.ParseDomain(host)
+	if err != nil {
+		return fmt.Errorf("invalid host: %v", err)
+	}
+
+	m.Lock()
+	defer m.Unlock()
+	if _, ok := m.hosts[d]; !ok {
+		return fmt.Errorf("%w: %q", errHostNotAllowed, d)
+	}
+	return nil
+}
+
+type dirCache autocert.DirCache
+
+func (d dirCache) Delete(ctx context.Context, name string) (rerr error) {
+	log := xlog.WithContext(ctx)
+	defer func() {
+		log.Debugx("dircache delete result", rerr, mlog.Field("name", name))
+	}()
+	err := autocert.DirCache(d).Delete(ctx, name)
+	if err != nil {
+		log.Errorx("deleting cert from dir cache", err, mlog.Field("name", name))
+	} else if !strings.HasSuffix(name, "+token") {
+		log.Info("autotls cert delete", mlog.Field("name", name))
+	}
+	return err
+}
+
+func (d dirCache) Get(ctx context.Context, name string) (rbuf []byte, rerr error) {
+	log := xlog.WithContext(ctx)
+	defer func() {
+		log.Debugx("dircache get result", rerr, mlog.Field("name", name))
+	}()
+	buf, err := autocert.DirCache(d).Get(ctx, name)
+	if err != nil && errors.Is(err, autocert.ErrCacheMiss) {
+		log.Infox("getting cert from dir cache", err, mlog.Field("name", name))
+	} else if err != nil {
+		log.Errorx("getting cert from dir cache", err, mlog.Field("name", name))
+	} else if !strings.HasSuffix(name, "+token") {
+		log.Debug("autotls cert get", mlog.Field("name", name))
+	}
+	return buf, err
+}
+
+func (d dirCache) Put(ctx context.Context, name string, data []byte) (rerr error) {
+	log := xlog.WithContext(ctx)
+	defer func() {
+		log.Debugx("dircache put result", rerr, mlog.Field("name", name))
+	}()
+	metricCertput.Inc()
+	err := autocert.DirCache(d).Put(ctx, name, data)
+	if err != nil {
+		log.Errorx("storing cert in dir cache", err, mlog.Field("name", name))
+	} else if !strings.HasSuffix(name, "+token") {
+		log.Info("autotls cert store", mlog.Field("name", name))
+	}
+	return err
+}

autotls/autotls_test.go

@@ -0,0 +1,97 @@
+package autotls
+
+import (
+	"context"
+	"errors"
+	"os"
+	"reflect"
+	"testing"
+
+	"golang.org/x/crypto/acme/autocert"
+
+	"github.com/mjl-/mox/dns"
+)
+
+func TestAutotls(t *testing.T) {
+	os.RemoveAll("../testdata/autotls")
+	os.MkdirAll("../testdata/autotls", 0770)
+
+	shutdown := make(chan struct{})
+	m, err := Load("test", "../testdata/autotls", "mox@localhost", "https://localhost/", shutdown)
+	if err != nil {
+		t.Fatalf("load manager: %v", err)
+	}
+	l := m.Hostnames()
+	if len(l) != 0 {
+		t.Fatalf("hostnames, got %v, expected empty list", l)
+	}
+	if err := m.HostPolicy(context.Background(), "mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) {
+		t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err)
+	}
+	m.AllowHostname(dns.Domain{ASCII: "mox.example"})
+	l = m.Hostnames()
+	if !reflect.DeepEqual(l, []dns.Domain{{ASCII: "mox.example"}}) {
+		t.Fatalf("hostnames, got %v, expected single mox.example", l)
+	}
+	if err := m.HostPolicy(context.Background(), "mox.example"); err != nil {
+		t.Fatalf("hostpolicy, got err %v, expected no error", err)
+	}
+	if err := m.HostPolicy(context.Background(), "other.mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) {
+		t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err)
+	}
+
+	ctx := context.Background()
+	cache := m.Manager.Cache
+	if _, err := cache.Get(ctx, "mox.example"); err == nil || !errors.Is(err, autocert.ErrCacheMiss) {
+		t.Fatalf("cache get for absent entry: got err %v, expected autocert.ErrCacheMiss", err)
+	}
+	if err := cache.Put(ctx, "mox.example", []byte("test")); err != nil {
+		t.Fatalf("cache put for absent entry: got err %v, expected error", err)
+	}
+	if data, err := cache.Get(ctx, "mox.example"); err != nil || string(data) != "test" {
+		t.Fatalf("cache get: got err %v data %q, expected nil, 'test'", err, data)
+	}
+	if err := cache.Put(ctx, "mox.example", []byte("test2")); err != nil {
+		t.Fatalf("cache put for absent entry: got err %v, expected error", err)
+	}
+	if data, err := cache.Get(ctx, "mox.example"); err != nil || string(data) != "test2" {
+		t.Fatalf("cache get: got err %v data %q, expected nil, 'test2'", err, data)
+	}
+	if err := cache.Delete(ctx, "mox.example"); err != nil {
+		t.Fatalf("cache delete: got err %v, expected no error", err)
+	}
+	if _, err := cache.Get(ctx, "mox.example"); err == nil || !errors.Is(err, autocert.ErrCacheMiss) {
+		t.Fatalf("cache get for absent entry: got err %v, expected autocert.ErrCacheMiss", err)
+	}
+
+	close(shutdown)
+	if err := m.HostPolicy(context.Background(), "mox.example"); err == nil {
+		t.Fatalf("hostpolicy, got err %v, expected error due to shutdown", err)
+	}
+
+	key0 := m.Manager.Client.Key
+
+	m, err = Load("test", "../testdata/autotls", "mox@localhost", "https://localhost/", shutdown)
+	if err != nil {
+		t.Fatalf("load manager again: %v", err)
+	}
+	if !reflect.DeepEqual(m.Manager.Client.Key, key0) {
+		t.Fatalf("private key changed after reload")
+	}
+	m.shutdown = make(chan struct{})
+	m.AllowHostname(dns.Domain{ASCII: "mox.example"})
+	if err := m.HostPolicy(context.Background(), "mox.example"); err != nil {
+		t.Fatalf("hostpolicy, got err %v, expected no error", err)
+	}
+
+	m2, err := Load("test2", "../testdata/autotls", "mox@localhost", "https://localhost/", shutdown)
+	if err != nil {
+		t.Fatalf("load another manager: %v", err)
+	}
+	if reflect.DeepEqual(m.Manager.Client.Key, m2.Manager.Client.Key) {
+		t.Fatalf("private key reused between managers")
+	}
+
+	// Only remove in case of success.
+	os.RemoveAll("../testdata/autotls")
+}

checkhtmljs

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec ./node_modules/.bin/jshint --extract always $@ | fixjshintlines

compatibility.txt

@@ -0,0 +1,3 @@
+Known compatibility issues.
+
+- Autodiscovery with Microsoft Outlook (on macOS): Outlook appears to use a Microsoft service to fetch the configuration, instead of connecting directly. Their service makes an invalid TLS handshake (an SNI name with a trailing dot), which is rejected by the Go crypto/tls library. (2023-01)

config/config.go

@@ -0,0 +1,245 @@
+package config
+
+import (
+	"crypto"
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"regexp"
+	"time"
+
+	"github.com/mjl-/mox/autotls"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/junk"
+	"github.com/mjl-/mox/mtasts"
+	"github.com/mjl-/mox/smtp"
+)
+
+// todo: better default values, so less has to be specified in the config file. junkfilter and rejects mailbox should be enabled by default. other features as well possibly.
+
+// Port returns port if non-zero, and fallback otherwise.
+func Port(port, fallback int) int {
+	if port == 0 {
+		return fallback
+	}
+	return port
+}
+
+// Static is a parsed form of the mox.conf configuration file, before converting it
+// into a mox.Config after additional processing.
+type Static struct {
+	DataDir          string            `sconf-doc:"Directory where all data is stored, e.g. queue, accounts and messages, ACME TLS certs/keys. If this is a relative path, it is relative to the directory of mox.conf."`
+	LogLevel         string            `sconf-doc:"Default log level, one of: error, info, debug, trace. Trace logs full SMTP and IMAP protocol transcripts, which can be a large amount of data."`
+	PackageLogLevels map[string]string `sconf:"optional" sconf-doc:"Overrides of log level per package (e.g. queue, smtpclient, smtpserver, imapserver, spf, dkim, dmarc, dmarcdb, autotls, junk, mtasts, tlsrpt)."`
+	Hostname         string            `sconf-doc:"Full hostname of system, e.g. mail.<domain>"`
+	HostnameDomain   dns.Domain        `sconf:"-" json:"-"` // Parsed form of hostname.
+	CheckUpdates     bool              `sconf:"optional" sconf-doc:"If enabled, a single DNS TXT lookup of _updates.xmox.nl is done every 24h to check for a new release. Each time a new release is found, a changelog is fetched from https://updates.xmox.nl and delivered to the postmaster mailbox."`
+	TLS              struct {
+		CA *struct {
+			AdditionalToSystem bool     `sconf:"optional"`
+			CertFiles          []string `sconf:"optional"`
+		} `sconf:"optional"`
+		CertPool *x509.CertPool `sconf:"-" json:"-"`
+	} `sconf:"optional" sconf-doc:"Global TLS configuration, e.g. for additional Certificate Authorities."`
+	ACME              map[string]ACME     `sconf:"optional" sconf-doc:"Automatic TLS configuration with ACME, e.g. through Let's Encrypt. The key is a name referenced in TLS configs, e.g. letsencrypt."`
+	AdminPasswordFile string              `sconf:"optional" sconf-doc:"File containing hash of admin password, for authentication in the web admin pages (if enabled)."`
+	Listeners         map[string]Listener `sconf-doc:"Listeners are groups of IP addresses and services enabled on those IP addresses, such as SMTP/IMAP or internal endpoints for administration or Prometheus metrics. All listeners with SMTP/IMAP services enabled will serve all configured domains."`
+	Postmaster        struct {
+		Account string
+		Mailbox string `sconf-doc:"E.g. Postmaster or Inbox."`
+	} `sconf-doc:"Destination for emails delivered to postmaster address."`
+	DefaultMailboxes []string `sconf:"optional" sconf-doc:"Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
+
+	// All IPs that were explicitly listen on for external SMTP. Only set when there
+	// are no unspecified external SMTP listeners and there is at most 1 for IPv4 and
+	// at most one for IPv6. Used for setting the local address when making outgoing
+	// connections. Those IPs are assumed to be in an SPF record for the domain,
+	// potentially unlike other IPs on the machine.  If there is only one address
+	// family, outgoing connections with the other address family are still made if
+	// possible.
+	SpecifiedSMTPListenIPs []net.IP `sconf:"-" json:"-"`
+}
+
+// Dynamic is the parsed form of domains.conf, and is automatically reloaded when changed.
+type Dynamic struct {
+	Domains  map[string]Domain  `sconf-doc:"Domains for which email is accepted. For internationalized domains, use their IDNA names in UTF-8."`
+	Accounts map[string]Account `sconf-doc:"Accounts to which email can be delivered. An account can accept email for multiple domains, for multiple localparts, and deliver to multiple mailboxes."`
+}
+
+type ACME struct {
+	DirectoryURL string        `sconf-doc:"For letsencrypt, use https://acme-v02.api.letsencrypt.org/directory."`
+	RenewBefore  time.Duration `sconf:"optional" sconf-doc:"How long before expiration to renew the certificate. Default is 30 days."`
+	ContactEmail string        `sconf-doc:"Email address to register at ACME provider. The provider can email you when certificates are about to expire. If you configure an address for which email is delivered by this server, keep in mind that TLS misconfigurations could result in such notification emails not arriving."`
+
+	Manager *autotls.Manager `sconf:"-" json:"-"`
+}
+
+type Listener struct {
+	IPs            []string   `sconf-doc:"Use 0.0.0.0 to listen on all IPv4 and/or :: to listen on all IPv6 addresses."`
+	Hostname       string     `sconf:"optional" sconf-doc:"If empty, the config global Hostname is used."`
+	HostnameDomain dns.Domain `sconf:"-" json:"-"` // Set when parsing config.
+
+	TLS                *TLS  `sconf:"optional" sconf-doc:"For SMTP/IMAP STARTTLS, direct TLS and HTTPS connections."`
+	SMTPMaxMessageSize int64 `sconf:"optional" sconf-doc:"Maximum size in bytes accepted incoming and outgoing messages. Default is 100MB."`
+	SMTP               struct {
+		Enabled         bool
+		Port            int          `sconf:"optional" sconf-doc:"Default 25."`
+		NoSTARTTLS      bool         `sconf:"optional" sconf-doc:"Do not offer STARTTLS to secure the connection. Not recommended."`
+		RequireSTARTTLS bool         `sconf:"optional" sconf-doc:"Do not accept incoming messages if STARTTLS is not active. Can be used in combination with a strict MTA-STS policy. A remote SMTP server may not support TLS and may not be able to deliver messages."`
+		DNSBLs          []string     `sconf:"optional" sconf-doc:"Addresses of DNS block lists for incoming messages. Block lists are only consulted for connections/messages without enough reputation to make an accept/reject decision. This prevents sending IPs of all communications to the block list provider. If any of the listed DNSBLs contains a requested IP address, the message is rejected as spam. The DNSBLs are checked for healthiness before use, at most once per 4 hours. Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net"`
+		DNSBLZones      []dns.Domain `sconf:"-"`
+	} `sconf:"optional"`
+	Submission struct {
+		Enabled           bool
+		Port              int  `sconf:"optional" sconf-doc:"Default 587."`
+		NoRequireSTARTTLS bool `sconf:"optional" sconf-doc:"Do not require STARTTLS. Since users must login, this means password may be sent without encryption. Not recommended."`
+	} `sconf:"optional" sconf-doc:"SMTP for submitting email, e.g. by email applications. Starts out in plain text, can be upgraded to TLS with the STARTTLS command. Prefer using Submissions which is always a TLS connection."`
+	Submissions struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Default 465."`
+	} `sconf:"optional" sconf-doc:"SMTP over TLS for submitting email, by email applications. Requires a TLS config."`
+	IMAP struct {
+		Enabled           bool
+		Port              int  `sconf:"optional" sconf-doc:"Default 143."`
+		NoRequireSTARTTLS bool `sconf:"optional" sconf-doc:"Enable this only when the connection is otherwise encrypted (e.g. through a VPN)."`
+	} `sconf:"optional" sconf-doc:"IMAP for reading email, by email applications. Starts out in plain text, can be upgraded to TLS with the STARTTLS command. Prefer using IMAPS instead which is always a TLS connection."`
+	IMAPS struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Default 993."`
+	} `sconf:"optional" sconf-doc:"IMAP over TLS for reading email, by email applications. Requires a TLS config."`
+	AdminHTTP struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Default 80."`
+	} `sconf:"optional" sconf-doc:"Admin web interface, for administrators and regular users wanting to change their password."`
+	AdminHTTPS struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Default 443."`
+	} `sconf:"optional" sconf-doc:"Admin web interface listener for HTTPS. Requires a TLS config."`
+	MetricsHTTP struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Default 8010."`
+	} `sconf:"optional" sconf-doc:"Serve prometheus metrics, for monitoring. You should not enable this on a public IP."`
+	PprofHTTP struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Default 8011."`
+	} `sconf:"optional" sconf-doc:"Serve /debug/pprof/ for profiling a running mox instance. Do not enable this on a public IP!"`
+	AutoconfigHTTPS struct {
+		Enabled bool
+	} `sconf:"optional" sconf-doc:"Serve autoconfiguration/autodiscovery to simplify configuring email applications, will use port 443. Requires a TLS config."`
+	MTASTSHTTPS struct {
+		Enabled bool
+	} `sconf:"optional" sconf-doc:"Serve MTA-STS policies describing SMTP TLS requirements, will use port 443. Requires a TLS config."`
+}
+
+type Domain struct {
+	Description                string  `sconf:"optional" sconf-doc:"Free-form description of domain."`
+	LocalpartCatchallSeparator string  `sconf:"optional" sconf-doc:"If not empty, only the string before the separator is used to for email delivery decisions. For example, if set to \"+\", you+anything@example.com will be delivered to you@example.com."`
+	LocalpartCaseSensitive     bool    `sconf:"optional" sconf-doc:"If set, upper/lower case is relevant for email delivery."`
+	DKIM                       DKIM    `sconf:"optional" sconf-doc:"With DKIM signing, a domain is taking responsibility for (content of) emails it sends, letting receiving mail servers build up a (hopefully positive) reputation of the domain, which can help with mail delivery."`
+	DMARC                      *DMARC  `sconf:"optional" sconf-doc:"With DMARC, a domain publishes, in DNS, a policy on how other mail servers should handle incoming messages with the From-header matching this domain and/or subdomain (depending on the configured alignment). Receiving mail servers use this to build up a reputation of this domain, which can help with mail delivery. A domain can also publish an email address to which reports about DMARC verification results can be sent by verifying mail servers, useful for monitoring. Incoming DMARC reports are automatically parsed, validated, added to metrics and stored in the reporting database for later display in the admin web pages."`
+	MTASTS                     *MTASTS `sconf:"optional" sconf-doc:"With MTA-STS a domain publishes, in DNS, presence of a policy for using/requiring TLS for SMTP connections. The policy is served over HTTPS."`
+	TLSRPT                     *TLSRPT `sconf:"optional" sconf-doc:"With TLSRPT a domain specifies in DNS where reports about encountered SMTP TLS behaviour should be sent. Useful for monitoring. Incoming TLS reports are automatically parsed, validated, added to metrics and stored in the reporting database for later display in the admin web pages."`
+
+	Domain dns.Domain `sconf:"-" json:"-"`
+}
+
+type DMARC struct {
+	Localpart string `sconf-doc:"Address-part before the @ that accepts DMARC reports. Must be non-internationalized. Recommended value: dmarc-reports."`
+	Account   string `sconf-doc:"Account to deliver to."`
+	Mailbox   string `sconf-doc:"Mailbox to deliver to, e.g. DMARC."`
+
+	ParsedLocalpart smtp.Localpart `sconf:"-"`
+}
+
+type MTASTS struct {
+	PolicyID string        `sconf-doc:"Policies are versioned. The version must be specified in the DNS record. If you change a policy, first change it in mox, then update the DNS record."`
+	Mode     mtasts.Mode   `sconf-doc:"testing, enforce or none. If set to enforce, a remote SMTP server will not deliver email to us if it cannot make a TLS connection."`
+	MaxAge   time.Duration `sconf-doc:"How long a remote mail server is allowed to cache a policy. Typically 1 or several weeks."`
+	MX       []string      `sconf:"optional" sconf-doc:"List of server names allowed for SMTP. If empty, the configured hostname is set. Host names can contain a wildcard (*) as a leading label (matching a single label, e.g. *.example matches host.example, not sub.host.example)."`
+	// todo: parse mx as valid mtasts.Policy.MX, with dns.ParseDomain but taking wildcard into account
+}
+
+type TLSRPT struct {
+	Localpart string `sconf-doc:"Address-part before the @ that accepts TLSRPT reports. Recommended value: tls-reports."`
+	Account   string `sconf-doc:"Account to deliver to."`
+	Mailbox   string `sconf-doc:"Mailbox to deliver to, e.g. TLSRPT."`
+
+	ParsedLocalpart smtp.Localpart `sconf:"-"`
+}
+
+type Selector struct {
+	Hash             string `sconf:"optional" sconf-doc:"sha256 (default) or (older, not recommended) sha1"`
+	HashEffective    string `sconf:"-"`
+	Canonicalization struct {
+		HeaderRelaxed bool `sconf-doc:"If set, some modifications to the headers (mostly whitespace) are allowed."`
+		BodyRelaxed   bool `sconf-doc:"If set, some whitespace modifications to the message body are allowed."`
+	} `sconf:"optional"`
+	Headers          []string `sconf:"optional" sconf-doc:"Headers to sign with DKIM. If empty, a reasonable default set of headers is selected."`
+	HeadersEffective []string `sconf:"-"`
+	DontSealHeaders  bool     `sconf:"optional" sconf-doc:"If set, don't prevent duplicate headers from being added. Not recommended."`
+	Expiration       string   `sconf:"optional" sconf-doc:"Period a signature is valid after signing, as duration, e.g. 72h. The period should be enough for delivery at the final destination, potentially with several hops/relays. In the order of days at least."`
+	PrivateKeyFile   string   `sconf-doc:"Either an RSA or ed25519 private key file in PKCS8 PEM form."`
+
+	ExpirationSeconds int           `sconf:"-" json:"-"` // Parsed from Expiration.
+	Key               crypto.Signer `sconf:"-" json:"-"` // As parsed with x509.ParsePKCS8PrivateKey.
+	Domain            dns.Domain    `sconf:"-" json:"-"` // Of selector only, not FQDN.
+}
+
+type DKIM struct {
+	Selectors map[string]Selector `sconf-doc:"Emails can be DKIM signed. Config parameters are per selector. A DNS record must be created for each selector. Add the name to Sign to use the selector for signing messages."`
+	Sign      []string            `sconf:"optional" sconf-doc:"List of selectors that emails will be signed with."`
+}
+
+type Account struct {
+	Domain       string                 `sconf-doc:"Default domain for addresses specified in Destinations. An address can specify a domain override."`
+	Description  string                 `sconf:"optional" sconf-doc:"Free form description, e.g. full name or alternative contact info."`
+	Destinations map[string]Destination `sconf-doc:"Destinations, specified as (encoded) localpart for Domain, or a full address including domain override."`
+	SubjectPass  struct {
+		Period time.Duration `sconf-doc:"How long unique values are accepted after generating, e.g. 12h."` // todo: have a reasonable default for this?
+	} `sconf:"optional" sconf-doc:"If configured, messages classified as weakly spam are rejected with instructions to retry delivery, but this time with a signed token added to the subject. During the next delivery attempt, the signed token will bypass the spam filter. Messages with a clear spam signal, such as a known bad reputation, are rejected/delayed without a signed token."`
+	RejectsMailbox string      `sconf:"optional" sconf-doc:"Mail that looks like spam will be rejected, but a copy can be stored temporarily in a mailbox, e.g. Rejects. If mail isn't coming in when you expect, you can look there. The mail still isn't accepted, so the remote mail server may retry (hopefully, if legitimate), or give up (hopefully, if indeed a spammer)."`
+	JunkFilter     *JunkFilter `sconf:"optional" sconf-doc:"Content-based filtering, using the junk-status of individual messages to rank words in such messages as spam or ham. It is recommended you always set the applicable (non)-junk status on messages, and that you do not empty your Trash because those messages contain valuable ham/spam training information."` // todo: sane defaults for junkfilter
+
+	DNSDomain dns.Domain `sconf:"-"` // Parsed form of Domain.
+}
+
+type JunkFilter struct {
+	Threshold float64 `sconf-doc:"Approximate spaminess score between 0 and 1 above which emails are rejected as spam. Each delivery attempt adds a little noise to make it slightly harder for spammers to identify words that strongly indicate non-spaminess and use it to bypass the filter. E.g. 0.95."`
+	junk.Params
+}
+
+type Destination struct {
+	Mailbox  string    `sconf:"optional" sconf-doc:"Mailbox to deliver to if none of Rulesets match. Default: Inbox."`
+	Rulesets []Ruleset `sconf:"optional" sconf-doc:"Delivery rules based on message and SMTP transaction. You may want to match each mailing list by SMTP MailFrom address, VerifiedDomain and/or List-ID header (typically <listname.example.org> if the list address is listname@example.org), delivering them to their own mailbox."`
+
+	DMARCReports bool `sconf:"-" json:"-"`
+	TLSReports   bool `sconf:"-" json:"-"`
+}
+
+type Ruleset struct {
+	SMTPMailFromRegexp string            `sconf:"optional" sconf-doc:"Matches if this regular expression matches (a substring of) the SMTP MAIL FROM address (not the message From-header). E.g. user@example.org."`
+	VerifiedDomain     string            `sconf:"optional" sconf-doc:"Matches if this domain or a subdomain matches a SPF- and/or DKIM-verified domain."`
+	HeadersRegexp      map[string]string `sconf:"optional" sconf-doc:"Matches if these header field/value regular expressions all match (substrings of) the message headers. Header fields and valuees are converted to lower case before matching. Whitespace is trimmed from the value before matching. A header field can occur multiple times in a message, only one instance has to match."`
+	// todo: add a SMTPRcptTo check, and MessageFrom that works on a properly parsed From header.
+
+	ListAllowDomain string `sconf:"optional" sconf-doc:"Influence the spam filtering, this does not change whether this ruleset applies to a message. If this domain matches an SPF- and/or DKIM-verified (sub)domain, the message is accepted without further spam checks, such as a junk filter or DMARC reject evaluation. DMARC rejects should not apply for mailing lists that are not configured to rewrite the From-header of messages that don't have a passing DKIM signature of the From-domain. Otherwise, by rejecting messages, you may be automatically unsubscribed from the mailing list."`
+
+	Mailbox string `sconf-doc:"Mailbox to deliver to if Rules match."`
+
+	SMTPMailFromRegexpCompiled *regexp.Regexp      `sconf:"-" json:"-"`
+	VerifiedDNSDomain          dns.Domain          `sconf:"-"`
+	HeadersRegexpCompiled      [][2]*regexp.Regexp `sconf:"-" json:"-"`
+	ListAllowDNSDomain         dns.Domain          `sconf:"-"`
+}
+
+type TLS struct {
+	ACME     string `sconf:"optional" sconf-doc:"Name of provider from top-level configuration to use for ACME, e.g. letsencrypt."`
+	KeyCerts []struct {
+		CertFile string `sconf-doc:"Certificate including intermediate CA certificates, in PEM format."`
+		KeyFile  string `sconf-doc:"Private key for certificate, in PEM format. PKCS8 is recommended, but PKCS1 and EC private keys are recognized as well."`
+	} `sconf:"optional"`
+	MinVersion string `sconf:"optional" sconf-doc:"Minimum TLS version. Default: TLSv1.2."`
+
+	Config     *tls.Config `sconf:"-" json:"-"`
+	ACMEConfig *tls.Config `sconf:"-" json:"-"`
+}

config/doc.go

@@ -0,0 +1,465 @@
+/*
+Package config holds the configuration file definitions for mox.conf (Static)
+and domains.conf (Dynamic).
+
+Annotated empty/default configuration files you could use as a starting point
+for your mox.conf and domains.conf, as generated by "mox config
+describe-static" and "mox config describe-domains":
+
+# mox.conf
+
+	# Directory where all data is stored, e.g. queue, accounts and messages, ACME TLS
+	# certs/keys. If this is a relative path, it is relative to the directory of
+	# mox.conf.
+	DataDir:
+
+	# Default log level, one of: error, info, debug, trace. Trace logs full SMTP and
+	# IMAP protocol transcripts, which can be a large amount of data.
+	LogLevel:
+
+	# Overrides of log level per package (e.g. queue, smtpclient, smtpserver,
+	# imapserver, spf, dkim, dmarc, dmarcdb, autotls, junk, mtasts, tlsrpt).
+	# (optional)
+	PackageLogLevels:
+		x:
+
+	# Full hostname of system, e.g. mail.<domain>
+	Hostname:
+
+	# If enabled, a single DNS TXT lookup of _updates.xmox.nl is done every 24h to
+	# check for a new release. Each time a new release is found, a changelog is
+	# fetched from https://updates.xmox.nl and delivered to the postmaster mailbox.
+	# (optional)
+	CheckUpdates: false
+
+	# Global TLS configuration, e.g. for additional Certificate Authorities.
+	# (optional)
+	TLS:
+
+		# (optional)
+		CA:
+
+			# (optional)
+			AdditionalToSystem: false
+
+			# (optional)
+			CertFiles:
+				-
+
+	# Automatic TLS configuration with ACME, e.g. through Let's Encrypt. The key is a
+	# name referenced in TLS configs, e.g. letsencrypt. (optional)
+	ACME:
+		x:
+
+			# For letsencrypt, use https://acme-v02.api.letsencrypt.org/directory.
+			DirectoryURL:
+
+			# How long before expiration to renew the certificate. Default is 30 days.
+			# (optional)
+			RenewBefore: 0s
+
+			# Email address to register at ACME provider. The provider can email you when
+			# certificates are about to expire. If you configure an address for which email is
+			# delivered by this server, keep in mind that TLS misconfigurations could result
+			# in such notification emails not arriving.
+			ContactEmail:
+
+	# File containing hash of admin password, for authentication in the web admin
+	# pages (if enabled). (optional)
+	AdminPasswordFile:
+
+	# Listeners are groups of IP addresses and services enabled on those IP addresses,
+	# such as SMTP/IMAP or internal endpoints for administration or Prometheus
+	# metrics. All listeners with SMTP/IMAP services enabled will serve all configured
+	# domains.
+	Listeners:
+		x:
+
+			# Use 0.0.0.0 to listen on all IPv4 and/or :: to listen on all IPv6 addresses.
+			IPs:
+				-
+
+			# If empty, the config global Hostname is used. (optional)
+			Hostname:
+
+			# For SMTP/IMAP STARTTLS, direct TLS and HTTPS connections. (optional)
+			TLS:
+
+				# Name of provider from top-level configuration to use for ACME, e.g. letsencrypt.
+				# (optional)
+				ACME:
+
+				# (optional)
+				KeyCerts:
+					-
+
+						# Certificate including intermediate CA certificates, in PEM format.
+						CertFile:
+
+						# Private key for certificate, in PEM format. PKCS8 is recommended, but PKCS1 and
+						# EC private keys are recognized as well.
+						KeyFile:
+
+				# Minimum TLS version. Default: TLSv1.2. (optional)
+				MinVersion:
+
+			# Maximum size in bytes accepted incoming and outgoing messages. Default is 100MB.
+			# (optional)
+			SMTPMaxMessageSize: 0
+
+			# (optional)
+			SMTP:
+				Enabled: false
+
+				# Default 25. (optional)
+				Port: 0
+
+				# Do not offer STARTTLS to secure the connection. Not recommended. (optional)
+				NoSTARTTLS: false
+
+				# Do not accept incoming messages if STARTTLS is not active. Can be used in
+				# combination with a strict MTA-STS policy. A remote SMTP server may not support
+				# TLS and may not be able to deliver messages. (optional)
+				RequireSTARTTLS: false
+
+				# Addresses of DNS block lists for incoming messages. Block lists are only
+				# consulted for connections/messages without enough reputation to make an
+				# accept/reject decision. This prevents sending IPs of all communications to the
+				# block list provider. If any of the listed DNSBLs contains a requested IP
+				# address, the message is rejected as spam. The DNSBLs are checked for healthiness
+				# before use, at most once per 4 hours. Example DNSBLs: sbl.spamhaus.org,
+				# bl.spamcop.net (optional)
+				DNSBLs:
+					-
+
+			# SMTP for submitting email, e.g. by email applications. Starts out in plain text,
+			# can be upgraded to TLS with the STARTTLS command. Prefer using Submissions which
+			# is always a TLS connection. (optional)
+			Submission:
+				Enabled: false
+
+				# Default 587. (optional)
+				Port: 0
+
+				# Do not require STARTTLS. Since users must login, this means password may be sent
+				# without encryption. Not recommended. (optional)
+				NoRequireSTARTTLS: false
+
+			# SMTP over TLS for submitting email, by email applications. Requires a TLS
+			# config. (optional)
+			Submissions:
+				Enabled: false
+
+				# Default 465. (optional)
+				Port: 0
+
+			# IMAP for reading email, by email applications. Starts out in plain text, can be
+			# upgraded to TLS with the STARTTLS command. Prefer using IMAPS instead which is
+			# always a TLS connection. (optional)
+			IMAP:
+				Enabled: false
+
+				# Default 143. (optional)
+				Port: 0
+
+				# Enable this only when the connection is otherwise encrypted (e.g. through a
+				# VPN). (optional)
+				NoRequireSTARTTLS: false
+
+			# IMAP over TLS for reading email, by email applications. Requires a TLS config.
+			# (optional)
+			IMAPS:
+				Enabled: false
+
+				# Default 993. (optional)
+				Port: 0
+
+			# Admin web interface, for administrators and regular users wanting to change
+			# their password. (optional)
+			AdminHTTP:
+				Enabled: false
+
+				# Default 80. (optional)
+				Port: 0
+
+			# Admin web interface listener for HTTPS. Requires a TLS config. (optional)
+			AdminHTTPS:
+				Enabled: false
+
+				# Default 443. (optional)
+				Port: 0
+
+			# Serve prometheus metrics, for monitoring. You should not enable this on a public
+			# IP. (optional)
+			MetricsHTTP:
+				Enabled: false
+
+				# Default 8010. (optional)
+				Port: 0
+
+			# Serve /debug/pprof/ for profiling a running mox instance. Do not enable this on
+			# a public IP! (optional)
+			PprofHTTP:
+				Enabled: false
+
+				# Default 8011. (optional)
+				Port: 0
+
+			# Serve autoconfiguration/autodiscovery to simplify configuring email
+			# applications, will use port 443. Requires a TLS config. (optional)
+			AutoconfigHTTPS:
+				Enabled: false
+
+			# Serve MTA-STS policies describing SMTP TLS requirements, will use port 443.
+			# Requires a TLS config. (optional)
+			MTASTSHTTPS:
+				Enabled: false
+
+	# Destination for emails delivered to postmaster address.
+	Postmaster:
+		Account:
+
+		# E.g. Postmaster or Inbox.
+		Mailbox:
+
+	# Mailboxes to create when adding an account. Inbox is always created. If no
+	# mailboxes are specified, the following are automatically created: Sent, Archive,
+	# Trash, Drafts and Junk. (optional)
+	DefaultMailboxes:
+		-
+
+# domains.conf
+
+	# Domains for which email is accepted. For internationalized domains, use their
+	# IDNA names in UTF-8.
+	Domains:
+		x:
+
+			# Free-form description of domain. (optional)
+			Description:
+
+			# If not empty, only the string before the separator is used to for email delivery
+			# decisions. For example, if set to "+", you+anything@example.com will be
+			# delivered to you@example.com. (optional)
+			LocalpartCatchallSeparator:
+
+			# If set, upper/lower case is relevant for email delivery. (optional)
+			LocalpartCaseSensitive: false
+
+			# With DKIM signing, a domain is taking responsibility for (content of) emails it
+			# sends, letting receiving mail servers build up a (hopefully positive) reputation
+			# of the domain, which can help with mail delivery. (optional)
+			DKIM:
+
+				# Emails can be DKIM signed. Config parameters are per selector. A DNS record must
+				# be created for each selector. Add the name to Sign to use the selector for
+				# signing messages.
+				Selectors:
+					x:
+
+						# sha256 (default) or (older, not recommended) sha1 (optional)
+						Hash:
+
+						# (optional)
+						Canonicalization:
+
+							# If set, some modifications to the headers (mostly whitespace) are allowed.
+							HeaderRelaxed: false
+
+							# If set, some whitespace modifications to the message body are allowed.
+							BodyRelaxed: false
+
+						# Headers to sign with DKIM. If empty, a reasonable default set of headers is
+						# selected. (optional)
+						Headers:
+							-
+
+						# If set, don't prevent duplicate headers from being added. Not recommended.
+						# (optional)
+						DontSealHeaders: false
+
+						# Period a signature is valid after signing, as duration, e.g. 72h. The period
+						# should be enough for delivery at the final destination, potentially with several
+						# hops/relays. In the order of days at least. (optional)
+						Expiration:
+
+						# Either an RSA or ed25519 private key file in PKCS8 PEM form.
+						PrivateKeyFile:
+
+				# List of selectors that emails will be signed with. (optional)
+				Sign:
+					-
+
+			# With DMARC, a domain publishes, in DNS, a policy on how other mail servers
+			# should handle incoming messages with the From-header matching this domain and/or
+			# subdomain (depending on the configured alignment). Receiving mail servers use
+			# this to build up a reputation of this domain, which can help with mail delivery.
+			# A domain can also publish an email address to which reports about DMARC
+			# verification results can be sent by verifying mail servers, useful for
+			# monitoring. Incoming DMARC reports are automatically parsed, validated, added to
+			# metrics and stored in the reporting database for later display in the admin web
+			# pages. (optional)
+			DMARC:
+
+				# Address-part before the @ that accepts DMARC reports. Must be
+				# non-internationalized. Recommended value: dmarc-reports.
+				Localpart:
+
+				# Account to deliver to.
+				Account:
+
+				# Mailbox to deliver to, e.g. DMARC.
+				Mailbox:
+
+			# With MTA-STS a domain publishes, in DNS, presence of a policy for
+			# using/requiring TLS for SMTP connections. The policy is served over HTTPS.
+			# (optional)
+			MTASTS:
+
+				# Policies are versioned. The version must be specified in the DNS record. If you
+				# change a policy, first change it in mox, then update the DNS record.
+				PolicyID:
+
+				# testing, enforce or none. If set to enforce, a remote SMTP server will not
+				# deliver email to us if it cannot make a TLS connection.
+				Mode:
+
+				# How long a remote mail server is allowed to cache a policy. Typically 1 or
+				# several weeks.
+				MaxAge: 0s
+
+				# List of server names allowed for SMTP. If empty, the configured hostname is set.
+				# Host names can contain a wildcard (*) as a leading label (matching a single
+				# label, e.g. *.example matches host.example, not sub.host.example). (optional)
+				MX:
+					-
+
+			# With TLSRPT a domain specifies in DNS where reports about encountered SMTP TLS
+			# behaviour should be sent. Useful for monitoring. Incoming TLS reports are
+			# automatically parsed, validated, added to metrics and stored in the reporting
+			# database for later display in the admin web pages. (optional)
+			TLSRPT:
+
+				# Address-part before the @ that accepts TLSRPT reports. Recommended value:
+				# tls-reports.
+				Localpart:
+
+				# Account to deliver to.
+				Account:
+
+				# Mailbox to deliver to, e.g. TLSRPT.
+				Mailbox:
+
+	# Accounts to which email can be delivered. An account can accept email for
+	# multiple domains, for multiple localparts, and deliver to multiple mailboxes.
+	Accounts:
+		x:
+
+			# Default domain for addresses specified in Destinations. An address can specify a
+			# domain override.
+			Domain:
+
+			# Free form description, e.g. full name or alternative contact info. (optional)
+			Description:
+
+			# Destinations, specified as (encoded) localpart for Domain, or a full address
+			# including domain override.
+			Destinations:
+				x:
+
+					# Mailbox to deliver to if none of Rulesets match. Default: Inbox. (optional)
+					Mailbox:
+
+					# Delivery rules based on message and SMTP transaction. You may want to match each
+					# mailing list by SMTP MailFrom address, VerifiedDomain and/or List-ID header
+					# (typically <listname.example.org> if the list address is listname@example.org),
+					# delivering them to their own mailbox. (optional)
+					Rulesets:
+						-
+
+							# Matches if this regular expression matches (a substring of) the SMTP MAIL FROM
+							# address (not the message From-header). E.g. user@example.org. (optional)
+							SMTPMailFromRegexp:
+
+							# Matches if this domain or a subdomain matches a SPF- and/or DKIM-verified
+							# domain. (optional)
+							VerifiedDomain:
+
+							# Matches if these header field/value regular expressions all match (substrings
+							# of) the message headers. Header fields and valuees are converted to lower case
+							# before matching. Whitespace is trimmed from the value before matching. A header
+							# field can occur multiple times in a message, only one instance has to match.
+							# (optional)
+							HeadersRegexp:
+								x:
+
+							# Influence the spam filtering, this does not change whether this ruleset applies
+							# to a message. If this domain matches an SPF- and/or DKIM-verified (sub)domain,
+							# the message is accepted without further spam checks, such as a junk filter or
+							# DMARC reject evaluation. DMARC rejects should not apply for mailing lists that
+							# are not configured to rewrite the From-header of messages that don't have a
+							# passing DKIM signature of the From-domain. Otherwise, by rejecting messages, you
+							# may be automatically unsubscribed from the mailing list. (optional)
+							ListAllowDomain:
+
+							# Mailbox to deliver to if Rules match.
+							Mailbox:
+
+			# If configured, messages classified as weakly spam are rejected with instructions
+			# to retry delivery, but this time with a signed token added to the subject.
+			# During the next delivery attempt, the signed token will bypass the spam filter.
+			# Messages with a clear spam signal, such as a known bad reputation, are
+			# rejected/delayed without a signed token. (optional)
+			SubjectPass:
+
+				# How long unique values are accepted after generating, e.g. 12h.
+				Period: 0s
+
+			# Mail that looks like spam will be rejected, but a copy can be stored temporarily
+			# in a mailbox, e.g. Rejects. If mail isn't coming in when you expect, you can
+			# look there. The mail still isn't accepted, so the remote mail server may retry
+			# (hopefully, if legitimate), or give up (hopefully, if indeed a spammer).
+			# (optional)
+			RejectsMailbox:
+
+			# Content-based filtering, using the junk-status of individual messages to rank
+			# words in such messages as spam or ham. It is recommended you always set the
+			# applicable (non)-junk status on messages, and that you do not empty your Trash
+			# because those messages contain valuable ham/spam training information.
+			# (optional)
+			JunkFilter:
+
+				# Approximate spaminess score between 0 and 1 above which emails are rejected as
+				# spam. Each delivery attempt adds a little noise to make it slightly harder for
+				# spammers to identify words that strongly indicate non-spaminess and use it to
+				# bypass the filter. E.g. 0.95.
+				Threshold: 0.000000
+				Params:
+
+					# Track ham/spam ranking for single words. (optional)
+					Onegrams: false
+
+					# Track ham/spam ranking for each two consecutive words. (optional)
+					Twograms: false
+
+					# Track ham/spam ranking for each three consecutive words. (optional)
+					Threegrams: false
+
+					# Maximum power a word (combination) can have. If spaminess is 0.99, and max power
+					# is 0.1, spaminess of the word will be set to 0.9. Similar for ham words.
+					MaxPower: 0.000000
+
+					# Number of most spammy/hammy words to use for calculating probability. E.g. 10.
+					TopWords: 0
+
+					# Ignore words that are this much away from 0.5 haminess/spaminess. E.g. 0.1,
+					# causing word (combinations) of 0.4 to 0.6 to be ignored. (optional)
+					IgnoreWords: 0.000000
+
+					# Occurrences in word database until a word is considered rare and its influence
+					# in calculating probability reduced. E.g. 1 or 2. (optional)
+					RareWords: 0
+*/
+package config
+
+// NOTE: DO NOT EDIT, this file is generated by ../gendoc.sh.

ctl.go

@@ -0,0 +1,607 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/metrics"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/queue"
+	"github.com/mjl-/mox/smtp"
+	"github.com/mjl-/mox/store"
+)
+
+// ctl represents a connection to the ctl unix domain socket of a running mox instance.
+// ctl provides functions to read/write commands/responses/data streams.
+type ctl struct {
+	conn net.Conn
+	r    *bufio.Reader // Set for first reader.
+	x    any           // If set, errors are handled by calling panic(x) instead of log.Fatal.
+	log  *mlog.Log     // If set, along with x, logging is done here.
+}
+
+// xctl opens a ctl connection.
+func xctl() *ctl {
+	p := mox.DataDirPath("ctl")
+	conn, err := net.Dial("unix", p)
+	if err != nil {
+		log.Fatalf("connecting to control socket at %q: %v", p, err)
+	}
+	ctl := &ctl{conn: conn}
+	version := ctl.xread()
+	if version != "ctlv0" {
+		log.Fatalf("ctl protocol mismatch, got %q, expected ctlv0", version)
+	}
+	return ctl
+}
+
+// Interpret msg as an error.
+// If ctl.x is set, the string is also written to the ctl to be interpreted as error by the other party.
+func (c *ctl) xerror(msg string) {
+	if c.x == nil {
+		log.Fatalln(msg)
+	}
+	c.log.Debugx("ctl error", fmt.Errorf("%s", msg))
+	c.xwrite(msg)
+	panic(c.x)
+}
+
+// Check if err is not nil. If so, handle error through ctl.x or log.Fatal. If
+// ctl.x is set, the error string is written to ctl, to be interpreted as an error
+// by the command reading from ctl.
+func (c *ctl) xcheck(err error, msg string) {
+	if err == nil {
+		return
+	}
+	if c.x == nil {
+		log.Fatalf("%s: %s", msg, err)
+	}
+	c.log.Debugx(msg, err)
+	fmt.Fprintf(c.conn, "%s: %s\n", msg, err)
+	panic(c.x)
+}
+
+// Read a line and return it without trailing newline.
+func (c *ctl) xread() string {
+	if c.r == nil {
+		c.r = bufio.NewReader(c.conn)
+	}
+	line, err := c.r.ReadString('\n')
+	c.xcheck(err, "read from ctl")
+	return strings.TrimSuffix(line, "\n")
+}
+
+// Read a line. If not "ok", the string is interpreted as an error.
+func (c *ctl) xreadok() {
+	line := c.xread()
+	if line != "ok" {
+		c.xerror(line)
+	}
+}
+
+// Write a string, typically a command or parameter.
+func (c *ctl) xwrite(text string) {
+	_, err := fmt.Fprintln(c.conn, text)
+	c.xcheck(err, "write")
+}
+
+// Write "ok" to indicate success.
+func (c *ctl) xwriteok() {
+	c.xwrite("ok")
+}
+
+// Copy data from a stream from ctl to dst.
+func (c *ctl) xstreamto(dst io.Writer) {
+	_, err := io.Copy(dst, c.reader())
+	c.xcheck(err, "reading message")
+}
+
+// Copy data from src to a stream to ctl.
+func (c *ctl) xstreamfrom(src io.Reader) {
+	w := c.writer()
+	_, err := io.Copy(w, src)
+	c.xcheck(err, "copying")
+	w.xclose()
+}
+
+// Writer returns an io.Writer for a data stream to ctl.
+// When done writing, caller must call xclose to signal the end of the stream.
+// Behaviour of "x" is copied from ctl.
+func (c *ctl) writer() *ctlwriter {
+	return &ctlwriter{conn: c.conn, x: c.x, log: c.log}
+}
+
+// Reader returns an io.Reader for a data stream from ctl.
+// Behaviour of "x" is copied from ctl.
+func (c *ctl) reader() *ctlreader {
+	if c.r == nil {
+		c.r = bufio.NewReader(c.conn)
+	}
+	return &ctlreader{conn: c.conn, r: c.r, x: c.x, log: c.log}
+}
+
+/*
+Ctlwriter and ctlreader implement the writing and reading a data stream. They
+implement the io.Writer and io.Reader interface. In the protocol below each
+non-data message ends with a newline that is typically stripped when
+interpreting.
+
+Zero or more data transactions:
+
+	> "123" (for data size) or an error message
+	> data, 123 bytes
+	< "ok" or an error message
+
+Followed by a end of stream indicated by zero data bytes message:
+
+	> "0"
+*/
+
+type ctlwriter struct {
+	conn net.Conn // Ctl socket from which messages are read.
+	buf  []byte   // Scratch buffer, for reading response.
+	x    any      // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal.
+	log  *mlog.Log
+}
+
+func (s *ctlwriter) Write(buf []byte) (int, error) {
+	_, err := fmt.Fprintf(s.conn, "%d\n", len(buf))
+	s.xcheck(err, "write count")
+	_, err = s.conn.Write(buf)
+	s.xcheck(err, "write data")
+	if s.buf == nil {
+		s.buf = make([]byte, 512)
+	}
+	n, err := s.conn.Read(s.buf)
+	s.xcheck(err, "reading response to write")
+	line := strings.TrimSuffix(string(s.buf[:n]), "\n")
+	if line != "ok" {
+		s.xerror(line)
+	}
+	return len(buf), nil
+}
+
+func (s *ctlwriter) xerror(msg string) {
+	if s.x == nil {
+		log.Fatalln(msg)
+	} else {
+		s.log.Debugx("error", fmt.Errorf("%s", msg))
+		panic(s.x)
+	}
+}
+
+func (s *ctlwriter) xcheck(err error, msg string) {
+	if err == nil {
+		return
+	}
+	if s.x == nil {
+		log.Fatalf("%s: %s", msg, err)
+	} else {
+		s.log.Debugx(msg, err)
+		panic(s.x)
+	}
+}
+
+func (s *ctlwriter) xclose() {
+	_, err := fmt.Fprintf(s.conn, "0\n")
+	s.xcheck(err, "write eof")
+}
+
+type ctlreader struct {
+	conn     net.Conn      // For writing "ok" after reading.
+	r        *bufio.Reader // Buffered ctl socket.
+	err      error         // If set, returned for each read. can also be io.EOF.
+	npending int           // Number of bytes that can still be read until a new count line must be read.
+	x        any           // If set, errors are handled with panic(x) instead of log.Fatal.
+	log      *mlog.Log     // If x is set, logging goes to log.
+}
+
+func (s *ctlreader) Read(buf []byte) (N int, Err error) {
+	if s.err != nil {
+		return 0, s.err
+	}
+	if s.npending == 0 {
+		line, err := s.r.ReadString('\n')
+		s.xcheck(err, "reading count")
+		line = strings.TrimSuffix(line, "\n")
+		n, err := strconv.ParseInt(line, 10, 32)
+		if err != nil {
+			s.xerror(line)
+		}
+		if n == 0 {
+			s.err = io.EOF
+			return 0, s.err
+		}
+		s.npending = int(n)
+		_, err = fmt.Fprintln(s.conn, "ok")
+		s.xcheck(err, "writing ok after reading")
+	}
+	rn := len(buf)
+	if rn > s.npending {
+		rn = s.npending
+	}
+	n, err := s.r.Read(buf[:rn])
+	s.xcheck(err, "read from ctl")
+	s.npending -= n
+	return n, err
+}
+
+func (s *ctlreader) xerror(msg string) {
+	if s.x == nil {
+		log.Fatalln(msg)
+	} else {
+		s.log.Debugx("error", fmt.Errorf("%s", msg))
+		panic(s.x)
+	}
+}
+
+func (s *ctlreader) xcheck(err error, msg string) {
+	if err == nil {
+		return
+	}
+	if s.x == nil {
+		log.Fatalf("%s: %s", msg, err)
+	} else {
+		s.log.Debugx(msg, err)
+		panic(s.x)
+	}
+}
+
+// servectl handles requests on the unix domain socket "ctl", e.g. for graceful shutdown, local mail delivery.
+func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) {
+	log.Debug("ctl connection")
+
+	var cmd string
+
+	var stop = struct{}{} // Sentinel value for panic and recover.
+	defer func() {
+		x := recover()
+		if x == nil || x == stop {
+			return
+		}
+		log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", cmd))
+		debug.PrintStack()
+		metrics.PanicInc("ctl")
+	}()
+
+	defer conn.Close()
+
+	ctl := &ctl{conn: conn, x: stop, log: log}
+	ctl.xwrite("ctlv0")
+
+	for {
+		servectlcmd(ctx, log, ctl, &cmd, shutdown)
+	}
+}
+
+func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shutdown func()) {
+	cmd := ctl.xread()
+	log.Info("ctl command", mlog.Field("cmd", cmd))
+	*xcmd = cmd
+	switch cmd {
+	case "stop":
+		shutdown()
+		os.Exit(0)
+
+	case "restart":
+		// First test the config.
+		_, errs := mox.ParseConfig(ctx, mox.ConfigStaticPath, true)
+		if len(errs) > 1 {
+			log.Error("multiple configuration errors before restart")
+			for _, err := range errs {
+				log.Errorx("config error", err)
+			}
+			ctl.xerror("restart aborted")
+		} else if len(errs) == 1 {
+			log.Errorx("configuration error, restart aborted", errs[0])
+			ctl.xerror(errs[0].Error())
+		}
+
+		binary, err := os.Executable()
+		ctl.xcheck(err, "finding executable")
+		cfr, ok := ctl.conn.(interface{ File() (*os.File, error) })
+		if !ok {
+			ctl.xerror("cannot dup ctl socket")
+		}
+		cf, err := cfr.File()
+		ctl.xcheck(err, "dup ctl socket")
+		defer cf.Close()
+		_, _, err = syscall.Syscall(syscall.SYS_FCNTL, cf.Fd(), syscall.F_SETFD, 0)
+		if err != syscall.Errno(0) {
+			ctl.xcheck(err, "clear close-on-exec on ctl socket")
+		}
+		ctl.xwriteok()
+
+		shutdown()
+
+		// todo future: we could gather all listen fd's, keep them open, passing them to the new process and indicate (in env var or cli flag) for which addresses they are, then exec and have the new process pick them up. not worth the trouble at the moment, our shutdown is typically quick enough.
+		// todo future: does this actually cleanup all M's on all platforms?
+
+		env := os.Environ()
+		var found bool
+		envv := fmt.Sprintf("MOX_RESTART_CTL_SOCKET=%d", cf.Fd())
+		for i, s := range env {
+			if strings.HasPrefix(s, "MOX_RESTART_CTL_SOCKET=") {
+				found = true
+				env[i] = envv
+				break
+			}
+		}
+		if !found {
+			env = append(env, envv)
+		}
+		// On success, we never get here and "serve" will write the OK on the MOX_RESTART_CTL_SOCKET and close it.
+		err = syscall.Exec(binary, os.Args, env)
+		runtime.KeepAlive(cf)
+		ctl.xcheck(err, "exec")
+
+	case "deliver":
+		/* The protocol, double quoted are literals.
+
+		> "deliver"
+		> address
+		< "ok"
+		> stream
+		< "ok"
+		*/
+
+		to := ctl.xread()
+		a, addr, err := store.OpenEmail(to)
+		ctl.xcheck(err, "lookup destination address")
+
+		msgFile, err := store.CreateMessageTemp("ctl-deliver")
+		ctl.xcheck(err, "creating temporary message file")
+		defer func() {
+			if msgFile != nil {
+				if err := os.Remove(msgFile.Name()); err != nil {
+					log.Errorx("removing temporary message file", err, mlog.Field("path", msgFile.Name()))
+				}
+				msgFile.Close()
+			}
+		}()
+		mw := &message.Writer{Writer: msgFile}
+		ctl.xwriteok()
+
+		ctl.xstreamto(mw)
+		err = msgFile.Sync()
+		ctl.xcheck(err, "syncing message to storage")
+		msgPrefix := []byte{}
+		if !mw.HaveHeaders {
+			msgPrefix = []byte("\r\n\r\n")
+		}
+
+		m := &store.Message{
+			Received:  time.Now(),
+			Size:      int64(len(msgPrefix)) + mw.Size,
+			MsgPrefix: msgPrefix,
+		}
+
+		a.WithWLock(func() {
+			err := a.Deliver(log, addr, m, msgFile, true)
+			ctl.xcheck(err, "delivering message")
+			log.Info("message delivered through ctl", mlog.Field("to", to))
+		})
+
+		msgFile.Close()
+		msgFile = nil
+		err = a.Close()
+		ctl.xcheck(err, "closing account")
+		ctl.xwriteok()
+
+	case "queue":
+		/* protocol:
+		> "queue"
+		< "ok"
+		< stream
+		*/
+		qmsgs, err := queue.List()
+		ctl.xcheck(err, "listing queue")
+		ctl.xwriteok()
+
+		xw := ctl.writer()
+		fmt.Fprintln(xw, "queue:")
+		for _, qm := range qmsgs {
+			var lastAttempt string
+			if qm.LastAttempt != nil {
+				lastAttempt = time.Since(*qm.LastAttempt).Round(time.Second).String()
+			}
+			fmt.Fprintf(xw, "%5d %s from:%s@%s to:%s@%s next %s last %s error %q\n", qm.ID, qm.Queued.Format(time.RFC3339), qm.SenderLocalpart, qm.SenderDomain, qm.RecipientLocalpart, qm.RecipientDomain, -time.Since(qm.NextAttempt).Round(time.Second), lastAttempt, qm.LastError)
+		}
+		if len(qmsgs) == 0 {
+			fmt.Fprint(xw, "(empty)\n")
+		}
+		xw.xclose()
+
+	case "queuekick", "queuedrop":
+		/* protocol:
+		> "queuekick" or "queuedrop"
+		> id
+		> todomain
+		> recipient
+		< count
+		< "ok" or error
+		*/
+
+		idstr := ctl.xread()
+		todomain := ctl.xread()
+		recipient := ctl.xread()
+		id, err := strconv.ParseInt(idstr, 10, 64)
+		if err != nil {
+			ctl.xwrite("0")
+			ctl.xcheck(err, "parsing id")
+		}
+
+		var count int
+		if cmd == "queuekick" {
+			count, err = queue.Kick(id, todomain, recipient)
+			ctl.xcheck(err, "kicking queue")
+		} else {
+			count, err = queue.Drop(id, todomain, recipient)
+			ctl.xcheck(err, "dropping messages from queue")
+		}
+		ctl.xwrite(fmt.Sprintf("%d", count))
+		ctl.xwriteok()
+
+	case "queuedump":
+		/* protocol:
+		> "queuedump"
+		> id
+		< "ok" or error
+		< stream
+		*/
+
+		idstr := ctl.xread()
+		id, err := strconv.ParseInt(idstr, 10, 64)
+		if err != nil {
+			ctl.xcheck(err, "parsing id")
+		}
+		mr, err := queue.OpenMessage(id)
+		ctl.xcheck(err, "opening message")
+		defer mr.Close()
+		ctl.xwriteok()
+		ctl.xstreamfrom(mr)
+
+	case "importmaildir", "importmbox":
+		mbox := cmd == "importmbox"
+		importctl(ctl, mbox)
+
+	case "domainadd":
+		/* protocol:
+		> "domainadd"
+		> domain
+		> account
+		> localpart
+		< "ok" or error
+		*/
+		domain := ctl.xread()
+		account := ctl.xread()
+		localpart := ctl.xread()
+		d, err := dns.ParseDomain(domain)
+		ctl.xcheck(err, "parsing domain")
+		err = mox.DomainAdd(ctx, d, account, smtp.Localpart(localpart))
+		ctl.xcheck(err, "adding domain")
+		ctl.xwriteok()
+
+	case "domainrm":
+		/* protocol:
+		> "domainrm"
+		> domain
+		< "ok" or error
+		*/
+		domain := ctl.xread()
+		d, err := dns.ParseDomain(domain)
+		ctl.xcheck(err, "parsing domain")
+		err = mox.DomainRemove(ctx, d)
+		ctl.xcheck(err, "removing domain")
+		ctl.xwriteok()
+
+	case "accountadd":
+		/* protocol:
+		> "accountadd"
+		> account
+		> address
+		< "ok" or error
+		*/
+		account := ctl.xread()
+		address := ctl.xread()
+		err := mox.AccountAdd(ctx, account, address)
+		ctl.xcheck(err, "adding account")
+		ctl.xwriteok()
+
+	case "accountrm":
+		/* protocol:
+		> "accountrm"
+		> account
+		< "ok" or error
+		*/
+		account := ctl.xread()
+		err := mox.AccountRemove(ctx, account)
+		ctl.xcheck(err, "removing account")
+		ctl.xwriteok()
+
+	case "addressadd":
+		/* protocol:
+		> "addressadd"
+		> address
+		> account
+		< "ok" or error
+		*/
+		address := ctl.xread()
+		account := ctl.xread()
+		err := mox.AddressAdd(ctx, address, account)
+		ctl.xcheck(err, "adding address")
+		ctl.xwriteok()
+
+	case "addressrm":
+		/* protocol:
+		> "addressrm"
+		> address
+		< "ok" or error
+		*/
+		address := ctl.xread()
+		err := mox.AddressRemove(ctx, address)
+		ctl.xcheck(err, "removing address")
+		ctl.xwriteok()
+
+	case "loglevels":
+		/* protocol:
+		> "loglevels"
+		< "ok"
+		< stream
+		*/
+		ctl.xwriteok()
+		l := mox.Conf.LogLevels()
+		keys := []string{}
+		for k := range l {
+			keys = append(keys, k)
+		}
+		sort.Slice(keys, func(i, j int) bool {
+			return keys[i] < keys[j]
+		})
+		s := ""
+		for _, k := range keys {
+			ks := k
+			if ks == "" {
+				ks = "(default)"
+			}
+			s += ks + ": " + mlog.LevelStrings[l[k]] + "\n"
+		}
+		ctl.xstreamfrom(strings.NewReader(s))
+
+	case "setloglevels":
+		/* protocol:
+		> "setloglevels"
+		> pkg
+		> level
+		< "ok" or error
+		*/
+		pkg := ctl.xread()
+		levelstr := ctl.xread()
+		level, ok := mlog.Levels[levelstr]
+		if !ok {
+			ctl.xerror("bad level")
+		}
+		mox.Conf.SetLogLevel(pkg, level)
+		ctl.xwriteok()
+
+	default:
+		log.Info("unrecognized command", mlog.Field("cmd", cmd))
+		ctl.xwrite("unrecognized command")
+		return
+	}
+}

dkim/dkim.go

@@ -0,0 +1,849 @@
+// Package dkim (DomainKeys Identified Mail signatures, RFC 6376) signs and
+// verifies DKIM signatures.
+//
+// Signatures are added to email messages in DKIM-Signature headers. By signing a
+// message, a domain takes responsibility for the message. A message can have
+// signatures for multiple domains, and the domain does not necessarily have to
+// match a domain in a From header. Receiving mail servers can build a spaminess
+// reputation based on domains that signed the message, along with other
+// mechanisms.
+package dkim
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto"
+	"crypto/ed25519"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxio"
+	"github.com/mjl-/mox/publicsuffix"
+	"github.com/mjl-/mox/smtp"
+)
+
+var xlog = mlog.New("dkim")
+
+var (
+	metricDKIMSign = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mox_dkim_sign_total",
+			Help: "DKIM messages signings.",
+		},
+		[]string{
+			"key",
+		},
+	)
+	metricDKIMVerify = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "mox_dkim_verify_duration_seconds",
+			Help:    "DKIM verify, including lookup, duration and result.",
+			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
+		},
+		[]string{
+			"algorithm",
+			"status",
+		},
+	)
+)
+
+var timeNow = time.Now // Replaced during tests.
+
+// Status is the result of verifying a DKIM-Signature as described by RFC 8601,
+// "Message Header Field for Indicating Message Authentication Status".
+type Status string
+
+// ../rfc/8601:959 ../rfc/6376:1770 ../rfc/6376:2459
+
+const (
+	StatusNone      Status = "none"      // Message was not signed.
+	StatusPass      Status = "pass"      // Message was signed and signature was verified.
+	StatusFail      Status = "fail"      // Message was signed, but signature was invalid.
+	StatusPolicy    Status = "policy"    // Message was signed, but signature is not accepted by policy.
+	StatusNeutral   Status = "neutral"   // Message was signed, but the signature contains an error or could not be processed. This status is also used for errors not covered by other statuses.
+	StatusTemperror Status = "temperror" // Message could not be verified. E.g. because of DNS resolve error. A later attempt may succeed. A missing DNS record is treated as temporary error, a new key may not have propagated through DNS shortly after it was taken into use.
+	StatusPermerror Status = "permerror" // Message cannot be verified. E.g. when a required header field is absent or for invalid (combination of) parameters. Typically set if a DNS record does not allow the signature, e.g. due to algorithm mismatch or expiry.
+)
+
+// Lookup errors.
+var (
+	ErrNoRecord        = errors.New("dkim: no dkim dns record for selector and domain")
+	ErrMultipleRecords = errors.New("dkim: multiple dkim dns record for selector and domain")
+	ErrDNS             = errors.New("dkim: lookup of dkim dns record")
+	ErrSyntax          = errors.New("dkim: syntax error in dkim dns record")
+)
+
+// Signature verification errors.
+var (
+	ErrSigAlgMismatch          = errors.New("dkim: signature algorithm mismatch with dns record")
+	ErrHashAlgNotAllowed       = errors.New("dkim: hash algorithm not allowed by dns record")
+	ErrKeyNotForEmail          = errors.New("dkim: dns record not allowed for use with email")
+	ErrDomainIdentityMismatch  = errors.New("dkim: dns record disallows mismatch of domain (d=) and identity (i=)")
+	ErrSigExpired              = errors.New("dkim: signature has expired")
+	ErrHashAlgorithmUnknown    = errors.New("dkim: unknown hash algorithm")
+	ErrBodyhashMismatch        = errors.New("dkim: body hash does not match")
+	ErrSigVerify               = errors.New("dkim: signature verification failed")
+	ErrSigAlgorithmUnknown     = errors.New("dkim: unknown signature algorithm")
+	ErrCanonicalizationUnknown = errors.New("dkim: unknown canonicalization")
+	ErrHeaderMalformed         = errors.New("dkim: mail message header is malformed")
+	ErrFrom                    = errors.New("dkim: bad from headers")
+	ErrQueryMethod             = errors.New("dkim: no recognized query method")
+	ErrKeyRevoked              = errors.New("dkim: key has been revoked")
+	ErrTLD                     = errors.New("dkim: signed domain is top-level domain, above organizational domain")
+	ErrPolicy                  = errors.New("dkim: signature rejected by policy")
+	ErrWeakKey                 = errors.New("dkim: key is too weak, need at least 1024 bits for rsa")
+)
+
+// Result is the conclusion of verifying one DKIM-Signature header. An email can
+// have multiple signatures, each with different parameters.
+//
+// To decide what to do with a message, both the signature parameters and the DNS
+// TXT record have to be consulted.
+type Result struct {
+	Status Status
+	Sig    *Sig    // Parsed form of DKIM-Signature header. Can be nil for invalid DKIM-Signature header.
+	Record *Record // Parsed form of DKIM DNS record for selector and domain in Sig. Optional.
+	Err    error   // If Status is not StatusPass, this error holds the details and can be checked using errors.Is.
+}
+
+// todo: use some io.Writer to hash the body and the header.
+
+// Sign returns line(s) with DKIM-Signature headers, generated according to the configuration.
+func Sign(ctx context.Context, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) {
+	log := xlog.WithContext(ctx)
+	start := timeNow()
+	defer func() {
+		log.Debugx("dkim sign result", rerr, mlog.Field("localpart", localpart), mlog.Field("domain", domain), mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start)))
+	}()
+
+	hdrs, bodyOffset, err := parseHeaders(bufio.NewReader(&moxio.AtReader{R: msg}))
+	if err != nil {
+		return "", fmt.Errorf("%w: %s", ErrHeaderMalformed, err)
+	}
+	nfrom := 0
+	for _, h := range hdrs {
+		if h.lkey == "from" {
+			nfrom++
+		}
+	}
+	if nfrom != 1 {
+		return "", fmt.Errorf("%w: message has %d from headers, need exactly 1", ErrFrom, nfrom)
+	}
+
+	type hashKey struct {
+		simple bool   // Canonicalization.
+		hash   string // lower-case hash.
+	}
+
+	var bodyHashes = map[hashKey][]byte{}
+
+	for _, sign := range c.Sign {
+		sel := c.Selectors[sign]
+		sig := newSigWithDefaults()
+		sig.Version = 1
+		switch sel.Key.(type) {
+		case *rsa.PrivateKey:
+			sig.AlgorithmSign = "rsa"
+			metricDKIMSign.WithLabelValues("rsa").Inc()
+		case ed25519.PrivateKey:
+			sig.AlgorithmSign = "ed25519"
+			metricDKIMSign.WithLabelValues("ed25519").Inc()
+		default:
+			return "", fmt.Errorf("internal error, unknown pivate key %T", sel.Key)
+		}
+		sig.AlgorithmHash = sel.HashEffective
+		sig.Domain = domain
+		sig.Selector = sel.Domain
+		sig.Identity = &Identity{&localpart, domain}
+		sig.SignedHeaders = append([]string{}, sel.HeadersEffective...)
+		if !sel.DontSealHeaders {
+			// ../rfc/6376:2156
+			// Each time a header name is added to the signature, the next unused value is
+			// signed (in reverse order as they occur in the message). So we can add each
+			// header name as often as it occurs. But now we'll add the header names one
+			// additional time, preventing someone from adding one more header later on.
+			counts := map[string]int{}
+			for _, h := range hdrs {
+				counts[h.lkey]++
+			}
+			for _, h := range sel.HeadersEffective {
+				for j := counts[strings.ToLower(h)]; j > 0; j-- {
+					sig.SignedHeaders = append(sig.SignedHeaders, h)
+				}
+			}
+		}
+		sig.SignTime = timeNow().Unix()
+		if sel.ExpirationSeconds > 0 {
+			sig.ExpireTime = sig.SignTime + int64(sel.ExpirationSeconds)
+		}
+
+		sig.Canonicalization = "simple"
+		if sel.Canonicalization.HeaderRelaxed {
+			sig.Canonicalization = "relaxed"
+		}
+		sig.Canonicalization += "/"
+		if sel.Canonicalization.BodyRelaxed {
+			sig.Canonicalization += "relaxed"
+		} else {
+			sig.Canonicalization += "simple"
+		}
+
+		h, hok := algHash(sig.AlgorithmHash)
+		if !hok {
+			return "", fmt.Errorf("unrecognized hash algorithm %q", sig.AlgorithmHash)
+		}
+
+		// We must now first calculate the hash over the body. Then include that hash in a
+		// new DKIM-Signature header. Then hash that and the signed headers into a data
+		// hash. Then that hash is finally signed and the signature included in the new
+		// DKIM-Signature header.
+		// ../rfc/6376:1700
+
+		hk := hashKey{!sel.Canonicalization.BodyRelaxed, strings.ToLower(sig.AlgorithmHash)}
+		if bh, ok := bodyHashes[hk]; ok {
+			sig.BodyHash = bh
+		} else {
+			br := bufio.NewReader(&moxio.AtReader{R: msg, Offset: int64(bodyOffset)})
+			bh, err = bodyHash(h.New(), !sel.Canonicalization.BodyRelaxed, br)
+			if err != nil {
+				return "", err
+			}
+			sig.BodyHash = bh
+			bodyHashes[hk] = bh
+		}
+
+		sigh, err := sig.Header()
+		if err != nil {
+			return "", err
+		}
+		verifySig := []byte(strings.TrimSuffix(sigh, "\r\n"))
+
+		dh, err := dataHash(h.New(), !sel.Canonicalization.HeaderRelaxed, sig, hdrs, verifySig)
+		if err != nil {
+			return "", err
+		}
+
+		switch key := sel.Key.(type) {
+		case *rsa.PrivateKey:
+			sig.Signature, err = key.Sign(cryptorand.Reader, dh, h)
+			if err != nil {
+				return "", fmt.Errorf("signing data: %v", err)
+			}
+		case ed25519.PrivateKey:
+			// crypto.Hash(0) indicates data isn't prehashed (ed25519ph). We are using
+			// PureEdDSA to sign the sha256 hash. ../rfc/8463:123 ../rfc/8032:427
+			sig.Signature, err = key.Sign(cryptorand.Reader, dh, crypto.Hash(0))
+			if err != nil {
+				return "", fmt.Errorf("signing data: %v", err)
+			}
+		default:
+			return "", fmt.Errorf("unsupported private key type: %s", err)
+		}
+
+		sigh, err = sig.Header()
+		if err != nil {
+			return "", err
+		}
+		headers += sigh
+	}
+
+	return headers, nil
+}
+
+// Lookup looks up the DKIM TXT record and parses it.
+//
+// A requested record is <selector>._domainkey.<domain>. Exactly one valid DKIM
+// record should be present.
+func Lookup(ctx context.Context, resolver dns.Resolver, selector, domain dns.Domain) (rstatus Status, rrecord *Record, rtxt string, rerr error) {
+	log := xlog.WithContext(ctx)
+	start := timeNow()
+	defer func() {
+		log.Debugx("dkim lookup result", rerr, mlog.Field("selector", selector), mlog.Field("domain", domain), mlog.Field("status", rstatus), mlog.Field("record", rrecord), mlog.Field("duration", time.Since(start)))
+	}()
+
+	name := selector.ASCII + "._domainkey." + domain.ASCII + "."
+	records, err := dns.WithPackage(resolver, "dkim").LookupTXT(ctx, name)
+	if dns.IsNotFound(err) {
+		// ../rfc/6376:2608
+		// We must return StatusPermerror. We may want to return StatusTemperror because in
+		// practice someone will start using a new key before DNS changes have propagated.
+		return StatusPermerror, nil, "", fmt.Errorf("%w: dns name %q", ErrNoRecord, name)
+	} else if err != nil {
+		return StatusTemperror, nil, "", fmt.Errorf("%w: dns name %q: %s", ErrDNS, name, err)
+	}
+
+	// ../rfc/6376:2612
+	var status = StatusTemperror
+	var record *Record
+	var txt string
+	err = nil
+	for _, s := range records {
+		// We interpret ../rfc/6376:2621 to mean that a record that claims to be v=DKIM1,
+		// but isn't actually valid, results in a StatusPermFail. But a record that isn't
+		// claiming to be DKIM1 is ignored.
+		var r *Record
+		var isdkim bool
+		r, isdkim, err = ParseRecord(s)
+		if err != nil && isdkim {
+			return StatusPermerror, nil, txt, fmt.Errorf("%w: %s", ErrSyntax, err)
+		} else if err != nil {
+			// Hopefully the remote MTA admin discovers the configuration error and fix it for
+			// an upcoming delivery attempt, in case we rejected with temporary status.
+			status = StatusTemperror
+			err = fmt.Errorf("%w: not a dkim record: %s", ErrSyntax, err)
+			continue
+		}
+		// If there are multiple valid records, return a temporary error. Perhaps the error is fixed soon.
+		// ../rfc/6376:1609
+		// ../rfc/6376:2584
+		if record != nil {
+			return StatusTemperror, nil, "", fmt.Errorf("%w: dns name %q", ErrMultipleRecords, name)
+		}
+		record = r
+		txt = s
+		err = nil
+	}
+
+	if record == nil {
+		return status, nil, "", err
+	}
+	return StatusNeutral, record, txt, nil
+}
+
+// Verify parses the DKIM-Signature headers in a message and verifies each of them.
+//
+// If the headers of the message cannot be found, an error is returned.
+// Otherwise, each DKIM-Signature header is reflected in the returned results.
+//
+// NOTE: Verify does not check if the domain (d=) that signed the message is
+// the domain of the sender. The caller, e.g. through DMARC, should do this.
+//
+// If ignoreTestMode is true and the DKIM record is in test mode (t=y), a
+// verification failure is treated as actual failure. With ignoreTestMode
+// false, such verification failures are treated as if there is no signature by
+// returning StatusNone.
+func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy func(*Sig) error, r io.ReaderAt, ignoreTestMode bool) (results []Result, rerr error) {
+	log := xlog.WithContext(ctx)
+	start := timeNow()
+	defer func() {
+		duration := float64(time.Since(start)) / float64(time.Second)
+		for _, r := range results {
+			var alg string
+			if r.Sig != nil {
+				alg = r.Sig.Algorithm()
+			}
+			status := string(r.Status)
+			metricDKIMVerify.WithLabelValues(alg, status).Observe(duration)
+		}
+
+		if len(results) == 0 {
+			log.Debugx("dkim verify result", rerr, mlog.Field("smtputf8", smtputf8), mlog.Field("duration", time.Since(start)))
+		}
+		for _, result := range results {
+			log.Debugx("dkim verify result", result.Err, mlog.Field("smtputf8", smtputf8), mlog.Field("status", result.Status), mlog.Field("sig", result.Sig), mlog.Field("record", result.Record), mlog.Field("duration", time.Since(start)))
+		}
+	}()
+
+	hdrs, bodyOffset, err := parseHeaders(bufio.NewReader(&moxio.AtReader{R: r}))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrHeaderMalformed, err)
+	}
+
+	// todo: reuse body hashes and possibly verify signatures in parallel. and start the dns lookup immediately. ../rfc/6376:2697
+
+	for _, h := range hdrs {
+		if h.lkey != "dkim-signature" {
+			continue
+		}
+
+		sig, verifySig, err := parseSignature(h.raw, smtputf8)
+		if err != nil {
+			// ../rfc/6376:2503
+			err := fmt.Errorf("parsing DKIM-Signature header: %w", err)
+			results = append(results, Result{StatusPermerror, nil, nil, err})
+			continue
+		}
+
+		h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig)
+		if err != nil {
+			results = append(results, Result{StatusPermerror, nil, nil, err})
+			continue
+		}
+
+		// ../rfc/6376:2560
+		if err := policy(sig); err != nil {
+			err := fmt.Errorf("%w: %s", ErrPolicy, err)
+			results = append(results, Result{StatusPolicy, nil, nil, err})
+			continue
+		}
+
+		br := bufio.NewReader(&moxio.AtReader{R: r, Offset: int64(bodyOffset)})
+		status, txt, err := verifySignature(ctx, resolver, sig, h, canonHeaderSimple, canonDataSimple, hdrs, verifySig, br, ignoreTestMode)
+		results = append(results, Result{status, sig, txt, err})
+	}
+	return results, nil
+}
+
+// check if signature is acceptable.
+// Only looks at the signature parameters, not at the DNS record.
+func checkSignatureParams(ctx context.Context, sig *Sig) (hash crypto.Hash, canonHeaderSimple, canonBodySimple bool, rerr error) {
+	// "From" header is required, ../rfc/6376:2122 ../rfc/6376:2546
+	var from bool
+	for _, h := range sig.SignedHeaders {
+		if strings.EqualFold(h, "from") {
+			from = true
+			break
+		}
+	}
+	if !from {
+		return 0, false, false, fmt.Errorf(`%w: required "from" header not signed`, ErrFrom)
+	}
+
+	// ../rfc/6376:2550
+	if sig.ExpireTime >= 0 && sig.ExpireTime < timeNow().Unix() {
+		return 0, false, false, fmt.Errorf("%w: expiration time %q", ErrSigExpired, time.Unix(sig.ExpireTime, 0).Format(time.RFC3339))
+	}
+
+	// ../rfc/6376:2554
+	// ../rfc/6376:3284
+	// Refuse signatures that reach beyond declared scope. We use the existing
+	// publicsuffix.Lookup to lookup a fake subdomain of the signing domain. If this
+	// supposed subdomain is actually an organizational domain, the signing domain
+	// shouldn't be signing for its organizational domain.
+	subdom := sig.Domain
+	subdom.ASCII = "x." + subdom.ASCII
+	if subdom.Unicode != "" {
+		subdom.Unicode = "x." + subdom.Unicode
+	}
+	if orgDom := publicsuffix.Lookup(ctx, subdom); subdom.ASCII == orgDom.ASCII {
+		return 0, false, false, fmt.Errorf("%w: %s", ErrTLD, sig.Domain)
+	}
+
+	h, hok := algHash(sig.AlgorithmHash)
+	if !hok {
+		return 0, false, false, fmt.Errorf("%w: %q", ErrHashAlgorithmUnknown, sig.AlgorithmHash)
+	}
+
+	t := strings.SplitN(sig.Canonicalization, "/", 2)
+
+	switch strings.ToLower(t[0]) {
+	case "simple":
+		canonHeaderSimple = true
+	case "relaxed":
+	default:
+		return 0, false, false, fmt.Errorf("%w: header canonicalization %q", ErrCanonicalizationUnknown, sig.Canonicalization)
+	}
+
+	canon := "simple"
+	if len(t) == 2 {
+		canon = t[1]
+	}
+	switch strings.ToLower(canon) {
+	case "simple":
+		canonBodySimple = true
+	case "relaxed":
+	default:
+		return 0, false, false, fmt.Errorf("%w: body canonicalization %q", ErrCanonicalizationUnknown, sig.Canonicalization)
+	}
+
+	// We only recognize query method dns/txt, which is the default. ../rfc/6376:1268
+	if len(sig.QueryMethods) > 0 {
+		var dnstxt bool
+		for _, m := range sig.QueryMethods {
+			if strings.EqualFold(m, "dns/txt") {
+				dnstxt = true
+				break
+			}
+		}
+		if !dnstxt {
+			return 0, false, false, fmt.Errorf("%w: need dns/txt", ErrQueryMethod)
+		}
+	}
+
+	return h, canonHeaderSimple, canonBodySimple, nil
+}
+
+// lookup the public key in the DNS and verify the signature.
+func verifySignature(ctx context.Context, resolver dns.Resolver, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (Status, *Record, error) {
+	// ../rfc/6376:2604
+	status, record, _, err := Lookup(ctx, resolver, sig.Selector, sig.Domain)
+	if err != nil {
+		// todo: for temporary errors, we could pass on information so caller returns a 4.7.5 ecode, ../rfc/6376:2777
+		return status, nil, err
+	}
+	status, err = verifySignatureRecord(record, sig, hash, canonHeaderSimple, canonDataSimple, hdrs, verifySig, body, ignoreTestMode)
+	return status, record, err
+}
+
+// verify a DKIM signature given the record from dns and signature from the email message.
+func verifySignatureRecord(r *Record, sig *Sig, hash crypto.Hash, canonHeaderSimple, canonDataSimple bool, hdrs []header, verifySig []byte, body *bufio.Reader, ignoreTestMode bool) (rstatus Status, rerr error) {
+	if !ignoreTestMode {
+		// ../rfc/6376:1558
+		y := false
+		for _, f := range r.Flags {
+			if strings.EqualFold(f, "y") {
+				y = true
+				break
+			}
+		}
+		if y {
+			defer func() {
+				if rstatus != StatusPass {
+					rstatus = StatusNone
+				}
+			}()
+		}
+	}
+
+	// ../rfc/6376:2639
+	if len(r.Hashes) > 0 {
+		ok := false
+		for _, h := range r.Hashes {
+			if strings.EqualFold(h, sig.AlgorithmHash) {
+				ok = true
+				break
+			}
+		}
+		if !ok {
+			return StatusPermerror, fmt.Errorf("%w: dkim dns record expects one of %q, message uses %q", ErrHashAlgNotAllowed, strings.Join(r.Hashes, ","), sig.AlgorithmHash)
+		}
+	}
+
+	// ../rfc/6376:2651
+	if !strings.EqualFold(r.Key, sig.AlgorithmSign) {
+		return StatusPermerror, fmt.Errorf("%w: dkim dns record requires algorithm %q, message has %q", ErrSigAlgMismatch, r.Key, sig.AlgorithmSign)
+	}
+
+	// ../rfc/6376:2645
+	if r.PublicKey == nil {
+		return StatusPermerror, ErrKeyRevoked
+	} else if rsaKey, ok := r.PublicKey.(*rsa.PublicKey); ok && rsaKey.N.BitLen() < 1024 {
+		// todo: find a reference that supports this.
+		return StatusPermerror, ErrWeakKey
+	}
+
+	// ../rfc/6376:1541
+	if !r.ServiceAllowed("email") {
+		return StatusPermerror, ErrKeyNotForEmail
+	}
+	for _, t := range r.Flags {
+		// ../rfc/6376:1575
+		// ../rfc/6376:1805
+		if strings.EqualFold(t, "s") && sig.Identity != nil {
+			if sig.Identity.Domain.ASCII != sig.Domain.ASCII {
+				return StatusPermerror, fmt.Errorf("%w: i= identity domain %q must match d= domain %q", ErrDomainIdentityMismatch, sig.Domain.ASCII, sig.Identity.Domain.ASCII)
+			}
+		}
+	}
+
+	if sig.Length >= 0 {
+		// todo future: implement l= parameter in signatures. we don't currently allow this through policy check.
+		return StatusPermerror, fmt.Errorf("l= (length) parameter in signature not yet implemented")
+	}
+
+	// We first check the signature is with the claimed body hash is valid. Then we
+	// verify the body hash. In case of invalid signatures, we won't read the entire
+	// body.
+	// ../rfc/6376:1700
+	// ../rfc/6376:2656
+
+	dh, err := dataHash(hash.New(), canonHeaderSimple, sig, hdrs, verifySig)
+	if err != nil {
+		// Any error is likely an invalid header field in the message, hence permanent error.
+		return StatusPermerror, fmt.Errorf("calculating data hash: %w", err)
+	}
+
+	switch k := r.PublicKey.(type) {
+	case *rsa.PublicKey:
+		if err := rsa.VerifyPKCS1v15(k, hash, dh, sig.Signature); err != nil {
+			return StatusFail, fmt.Errorf("%w: rsa verification: %s", ErrSigVerify, err)
+		}
+	case ed25519.PublicKey:
+		if ok := ed25519.Verify(k, dh, sig.Signature); !ok {
+			return StatusFail, fmt.Errorf("%w: ed25519 verification", ErrSigVerify)
+		}
+	default:
+		return StatusPermerror, fmt.Errorf("%w: unrecognized signature algorithm %q", ErrSigAlgorithmUnknown, r.Key)
+	}
+
+	bh, err := bodyHash(hash.New(), canonDataSimple, body)
+	if err != nil {
+		// Any error is likely some internal error, hence temporary error.
+		return StatusTemperror, fmt.Errorf("calculating body hash: %w", err)
+	}
+	if !bytes.Equal(sig.BodyHash, bh) {
+		return StatusFail, fmt.Errorf("%w: signature bodyhash %x != calculated bodyhash %x", ErrBodyhashMismatch, sig.BodyHash, bh)
+	}
+
+	return StatusPass, nil
+}
+
+func algHash(s string) (crypto.Hash, bool) {
+	if strings.EqualFold(s, "sha1") {
+		return crypto.SHA1, true
+	} else if strings.EqualFold(s, "sha256") {
+		return crypto.SHA256, true
+	}
+	return 0, false
+}
+
+// bodyHash calculates the hash over the body.
+func bodyHash(h hash.Hash, canonSimple bool, body *bufio.Reader) ([]byte, error) {
+	// todo: take l= into account. we don't currently allow it for policy reasons.
+
+	var crlf = []byte("\r\n")
+
+	if canonSimple {
+		// ../rfc/6376:864, ensure body ends with exactly one trailing crlf.
+		ncrlf := 0
+		for {
+			buf, err := body.ReadBytes('\n')
+			if len(buf) == 0 && err == io.EOF {
+				break
+			}
+			if err != nil && err != io.EOF {
+				return nil, err
+			}
+			hascrlf := bytes.HasSuffix(buf, crlf)
+			if hascrlf {
+				buf = buf[:len(buf)-2]
+			}
+			if len(buf) > 0 {
+				for ; ncrlf > 0; ncrlf-- {
+					h.Write(crlf)
+				}
+				h.Write(buf)
+			}
+			if hascrlf {
+				ncrlf++
+			}
+		}
+		h.Write(crlf)
+	} else {
+		hb := bufio.NewWriter(h)
+
+		// We go through the body line by line, replacing WSP with a single space and removing whitespace at the end of lines.
+		// We stash "empty" lines. If they turn out to be at the end of the file, we must drop them.
+		stash := &bytes.Buffer{}
+		var line bool         // Whether buffer read is for continuation of line.
+		var prev byte         // Previous byte read for line.
+		linesEmpty := true    // Whether stash contains only empty lines and may need to be dropped.
+		var bodynonempty bool // Whether body is non-empty, for adding missing crlf.
+		var hascrlf bool      // Whether current/last line ends with crlf, for adding missing crlf.
+		for {
+			// todo: should not read line at a time, count empty lines. reduces max memory usage. a message with lots of empty lines can cause high memory use.
+			buf, err := body.ReadBytes('\n')
+			if len(buf) == 0 && err == io.EOF {
+				break
+			}
+			if err != nil && err != io.EOF {
+				return nil, err
+			}
+			bodynonempty = true
+
+			hascrlf = bytes.HasSuffix(buf, crlf)
+			if hascrlf {
+				buf = buf[:len(buf)-2]
+
+				// ../rfc/6376:893, "ignore all whitespace at the end of lines".
+				// todo: what is "whitespace"? it isn't WSP (space and tab), the next line mentions WSP explicitly for another rule. should we drop trailing \r, \n, \v, more?
+				buf = bytes.TrimRight(buf, " \t")
+			}
+
+			// Replace one or more WSP to a single SP.
+			for i, c := range buf {
+				wsp := c == ' ' || c == '\t'
+				if (i >= 0 || line) && wsp {
+					if prev == ' ' {
+						continue
+					}
+					prev = ' '
+					c = ' '
+				} else {
+					prev = c
+				}
+				if !wsp {
+					linesEmpty = false
+				}
+				stash.WriteByte(c)
+			}
+			if hascrlf {
+				stash.Write(crlf)
+			}
+			line = !hascrlf
+			if !linesEmpty {
+				hb.Write(stash.Bytes())
+				stash.Reset()
+				linesEmpty = true
+			}
+		}
+		// ../rfc/6376:886
+		// Only for non-empty bodies without trailing crlf do we add the missing crlf.
+		if bodynonempty && !hascrlf {
+			hb.Write(crlf)
+		}
+
+		hb.Flush()
+	}
+	return h.Sum(nil), nil
+}
+
+func dataHash(h hash.Hash, canonSimple bool, sig *Sig, hdrs []header, verifySig []byte) ([]byte, error) {
+	headers := ""
+	revHdrs := map[string][]header{}
+	for _, h := range hdrs {
+		revHdrs[h.lkey] = append([]header{h}, revHdrs[h.lkey]...)
+	}
+
+	for _, key := range sig.SignedHeaders {
+		lkey := strings.ToLower(key)
+		h := revHdrs[lkey]
+		if len(h) == 0 {
+			continue
+		}
+		revHdrs[lkey] = h[1:]
+		s := string(h[0].raw)
+		if canonSimple {
+			// ../rfc/6376:823
+			// Add unmodified.
+			headers += s
+		} else {
+			ch, err := relaxedCanonicalHeaderWithoutCRLF(s)
+			if err != nil {
+				return nil, fmt.Errorf("canonicalizing header: %w", err)
+			}
+			headers += ch + "\r\n"
+		}
+	}
+	// ../rfc/6376:2377, canonicalization does not apply to the dkim-signature header.
+	h.Write([]byte(headers))
+	dkimSig := verifySig
+	if !canonSimple {
+		ch, err := relaxedCanonicalHeaderWithoutCRLF(string(verifySig))
+		if err != nil {
+			return nil, fmt.Errorf("canonicalizing DKIM-Signature header: %w", err)
+		}
+		dkimSig = []byte(ch)
+	}
+	h.Write(dkimSig)
+	return h.Sum(nil), nil
+}
+
+// a single header, can be multiline.
+func relaxedCanonicalHeaderWithoutCRLF(s string) (string, error) {
+	// ../rfc/6376:831
+	t := strings.SplitN(s, ":", 2)
+	if len(t) != 2 {
+		return "", fmt.Errorf("%w: invalid header %q", ErrHeaderMalformed, s)
+	}
+
+	// Unfold, we keep the leading WSP on continuation lines and fix it up below.
+	v := strings.ReplaceAll(t[1], "\r\n", "")
+
+	// Replace one or more WSP to a single SP.
+	var nv []byte
+	var prev byte
+	for i, c := range []byte(v) {
+		if i >= 0 && c == ' ' || c == '\t' {
+			if prev == ' ' {
+				continue
+			}
+			prev = ' '
+			c = ' '
+		} else {
+			prev = c
+		}
+		nv = append(nv, c)
+	}
+
+	ch := strings.ToLower(strings.TrimRight(t[0], " \t")) + ":" + strings.Trim(string(nv), " \t")
+	return ch, nil
+}
+
+type header struct {
+	key   string // Key in original case.
+	lkey  string // Key in lower-case, for canonical case.
+	value []byte // Literal header value, possibly spanning multiple lines, not modified in any way, including crlf, excluding leading key and colon.
+	raw   []byte // Like value, but including original leading key and colon. Ready for use as simple header canonicalized use.
+}
+
+func parseHeaders(br *bufio.Reader) ([]header, int, error) {
+	var o int
+	var l []header
+	var key, lkey string
+	var value []byte
+	var raw []byte
+	for {
+		line, err := readline(br)
+		if err != nil {
+			return nil, 0, err
+		}
+		o += len(line)
+		if bytes.Equal(line, []byte("\r\n")) {
+			break
+		}
+		if line[0] == ' ' || line[0] == '\t' {
+			if len(l) == 0 && key == "" {
+				return nil, 0, fmt.Errorf("malformed message, starts with space/tab")
+			}
+			value = append(value, line...)
+			raw = append(raw, line...)
+			continue
+		}
+		if key != "" {
+			l = append(l, header{key, lkey, value, raw})
+		}
+		t := bytes.SplitN(line, []byte(":"), 2)
+		if len(t) != 2 {
+			return nil, 0, fmt.Errorf("malformed message, header without colon")
+		}
+
+		key = strings.TrimRight(string(t[0]), " \t") // todo: where is this specified?
+		// Check for valid characters. ../rfc/5322:1689 ../rfc/6532:193
+		for _, c := range key {
+			if c <= ' ' || c >= 0x7f {
+				return nil, 0, fmt.Errorf("invalid header field name")
+			}
+		}
+		if key == "" {
+			return nil, 0, fmt.Errorf("empty header key")
+		}
+		lkey = strings.ToLower(key)
+		value = append([]byte{}, t[1]...)
+		raw = append([]byte{}, line...)
+	}
+	if key != "" {
+		l = append(l, header{key, lkey, value, raw})
+	}
+	return l, o, nil
+}
+
+func readline(r *bufio.Reader) ([]byte, error) {
+	var buf []byte
+	for {
+		line, err := r.ReadBytes('\n')
+		if err != nil {
+			return nil, err
+		}
+		if bytes.HasSuffix(line, []byte("\r\n")) {
+			if len(buf) == 0 {
+				return line, nil
+			}
+			return append(buf, line...), nil
+		}
+		buf = append(buf, line...)
+	}
+}

dkim/dkim_test.go

@@ -0,0 +1,702 @@
+package dkim
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto"
+	"crypto/ed25519"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/dns"
+)
+
+func policyOK(sig *Sig) error {
+	return nil
+}
+
+func parseRSAKey(t *testing.T, rsaText string) *rsa.PrivateKey {
+	rsab, _ := pem.Decode([]byte(rsaText))
+	if rsab == nil {
+		t.Fatalf("no pem in privKey")
+	}
+
+	key, err := x509.ParsePKCS8PrivateKey(rsab.Bytes)
+	if err != nil {
+		t.Fatalf("parsing private key: %s", err)
+	}
+	return key.(*rsa.PrivateKey)
+}
+
+func getRSAKey(t *testing.T) *rsa.PrivateKey {
+	// Generated with:
+	// openssl genrsa -out pkcs1.pem 2048
+	// openssl pkcs8 -topk8 -inform pem -in pkcs1.pem -outform pem -nocrypt -out pkcs8.pem
+	const rsaText = `-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCu7iTF/AAvJQ3U
+WRlcXd+n6HXOSYvmDlqjLsuCKn6/T+Ma0ZtobCRfzyXh5pFQBCHffW6fpEzJs/2o
++e896zb1QKjD8Xxsjarjdw1iXzgMj/lhDGWyNyUHC34+k77UfpQBZgPLvZHyYyQG
+sVMzzmvURE+GMFmXYUiGI581PdCx4bNba/4gYQnc/eqQ8oX0T//2RdRqdhdDM2d7
+CYALtkxKetH1F+Rz7XDjFmI3GjPs1KwVdh+Cl8kejThi0SVxXpqnoqB2WGsr/lGG
+GxsxcpLb/+KWFjI0go3OJjMaxFCmhB0pGdW8I7kNwNrZsCdSvmjMDojNuegx6WMg
+/T7go3CvAgMBAAECggEAQA3AlmSDtr+lNDvZ7voKwwN6W6qPmRJpevZQG54u4iPA
+/5mAA/kRSqnh77mLPRb+RkU6RCeX3IXVXNIEGhKugZiHE5Sx4FfxmrAFzR8buXHg
+uXoeJOdPXiiFtilIh6u/y1FNE4YbUnud/fthgYdU8Zl/2x2KOMWtFj0l94tmhzOI
+b2y8/U8r85anI5XGYuzRCqKS1WskXhkXH8LZUB+9yAxX7V5ysgxjofM4FW8ns7yj
+K4cBS8KY2v3t7TZ4FgwkAhPcTfBc/E2UWT1Ztmr+18LFV5bqI8g2YlN+BgCxU7U/
+1tawxqFhs+xowEpzNwAvjAIPpptIRiY1rz7sBB9g5QKBgQDLo/5rTUwNOPR9dYvA
++DYUSCfxvNamI4GI66AgwOeN8O+W+dRDF/Ewbk/SJsBPSLIYzEiQ2uYKcNEmIjo+
+7WwSCJZjKujovw77s9JAHexhpd8uLD2w9l3KeTg41LEYm2uVwoXWEHYSYJ9Ynz0M
+PWxvi2Hm0IoQ7gJIfxng/wIw3QKBgQDb6GFvPH/OTs40+dopwtm3irmkBAmT8N0b
+3TpehONCOiL4GPxmn2DN6ELhHFV27Jj/1CfpGVbcBlaS1xYUGUGsB9gYukhdaBST
+KGHRoeZDcf0gaQLKG15EEfFOvcKI9aGljV8FdFfG+Z4fW3LA8khvpvjLLkv1A1jM
+MrEBthco+wKBgD45EM9GohtUMNh450gCT7voxFPICKphJP5qSNZZOyeS3BJ8qdAK
+a8cJndgvwQk4xDpxiSbBzBKaoD2Prc52i1QDTbhlbx9W6cQdEPxIaGb54PThzcPZ
+s5Tfbz9mNeq36qqq8mwTQZCh926D0YqA5jY7F6IITHeZ0hbGx2iJYuj9AoGARIyK
+ms8kE95y3wanX+8ySMmAlsT/a1NgyUfL4xzPbpyKvAWl4CN8XJMzDdL0PS8BfnXW
+vw28CrgbEojjg/5ff02uqf6fgiZoi3rCC0PJcGq++fRh/zhKyTNCokX6txDCg8Wu
+wheDKS40gRfTjJu5wrwsv8E9wjF546VFkf/99jMCgYEAm/x+kEfWKuzx8pQT66TY
+pxnC41upJOO1htTHNIN24J7XrrFI5+OZq90G+t/VgWX08Z8RlhejX+ukBf+SRu3u
+5VMGcAs4px+iECX/FHo21YQFnrmArN1zdFxPU3rBWoBueqmGO6FT0HBbKzTuS7N0
+7fIv3GQqImz3+ZbYWlXfkPI=
+-----END PRIVATE KEY-----`
+	return parseRSAKey(t, rsaText)
+}
+
+func getWeakRSAKey(t *testing.T) *rsa.PrivateKey {
+	const rsaText = `-----BEGIN PRIVATE KEY-----
+MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAsQo3ATJAZ4aAZz+l
+ndXl27ODOY+49DjYxwhgtg+OU8A1WEYCfWaZ7ozYtpsqH8GNFvlKtK38eKbdDuLw
+gsFYMQIDAQABAkBwstb2/P1Aqb9deoe8JOiw5eJYJySO2w0sDio6W0a4Cqi7XQ7r
+/yZ1gOp+ZnShX/sJq0Pd16UkJUUEtEPoZyptAiEA4KLP8pz/9R0t7Envqph1oVjQ
+CVDIL/UKRmdnMiwwDosCIQDJwiu08UgNNeliAygbkC2cdszjf4a3laGmYbfWrtAn
+swIgUBfc+w0degDgadpm2LWpY1DuRBQIfIjrE/U0Z0A4FkcCIHxEuoLycjygziTu
+aM/BWDac/cnKDIIbCbvfSEpU1iT9AiBsbkAcYCQ8mR77BX6gZKEc74nSce29gmR7
+mtrKWknTDQ==
+-----END PRIVATE KEY-----`
+	return parseRSAKey(t, rsaText)
+}
+
+func TestParseSignature(t *testing.T) {
+	// Domain name must always be A-labels, not U-labels. We do allow localpart with non-ascii.
+	hdr := `DKIM-Signature: v=1; a=rsa-sha256; d=xn--h-bga.mox.example; s=xn--yr2021-pua;
+        i=møx@xn--h-bga.mox.example; t=1643719203; h=From:To:Cc:Bcc:Reply-To:
+        References:In-Reply-To:Subject:Date:Message-ID:Content-Type:From:To:Subject:
+        Date:Message-ID:Content-Type;
+        bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=; b=dtgAOl71h/dNPQrmZTi3SBVkm+
+        EjMnF7sWGT123fa5g+m6nGpPue+I+067wwtkWQhsedbDkqT7gZb5WaG5baZsr9e/XpJ/iX4g6YXpr
+        07aLY8eF9jazcGcRCVCqLtyq0UJQ2Oz/ML74aYu1beh3jXsoI+k3fJ+0/gKSVC7enCFpNe1HhbXVS
+        4HRy/Rw261OEIy2e20lyPT4iDk2oODabzYa28HnXIciIMELjbc/sSawG68SAnhwdkWBrRzBDMCCHm
+        wvkmgDsVJWtdzjJqjxK2mYVxBMJT0lvsutXgYQ+rr6BLtjHsOb8GMSbQGzY5SJ3N8TP02pw5OykBu
+        B/aHff1A==
+`
+	smtputf8 := true
+	_, _, err := parseSignature([]byte(strings.ReplaceAll(hdr, "\n", "\r\n")), smtputf8)
+	if err != nil {
+		t.Fatalf("parsing signature: %s", err)
+	}
+}
+
+func TestVerifyRSA(t *testing.T) {
+	message := strings.ReplaceAll(`Return-Path: <mechiel@ueber.net>
+X-Original-To: mechiel@ueber.net
+Delivered-To: mechiel@ueber.net
+Received: from [IPV6:2a02:a210:4a3:b80:ca31:30ee:74a7:56e0] (unknown [IPv6:2a02:a210:4a3:b80:ca31:30ee:74a7:56e0])
+	by koriander.ueber.net (Postfix) with ESMTPSA id E119EDEB0B
+	for <mechiel@ueber.net>; Fri, 10 Dec 2021 20:09:08 +0100 (CET)
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=ueber.net;
+	s=koriander; t=1639163348;
+	bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=Date:To:From:Subject:From;
+	b=rpWruWprs2TB7/MnulA2n2WtfUIfrrnAvRoSrip1ruX5ORN4AOYPPMmk/gGBDdc6O
+	 grRpSsNzR9BrWcooYfbNfSbl04nPKMp0acsZGfpvkj0+mqk5b8lqZs3vncG1fHlQc7
+	 0KXfnAHyEs7bjyKGbrw2XG1p/EDoBjIjUsdpdCAtamMGv3A3irof81oSqvwvi2KQks
+	 17aB1YAL9Xzkq9ipo1aWvDf2W6h6qH94YyNocyZSVJ+SlVm3InNaF8APkV85wOm19U
+	 9OW81eeuQbvSPcQZJVOmrWzp7XKHaXH0MYE3+hdH/2VtpCnPbh5Zj9SaIgVbaN6NPG
+	 Ua0E07rwC86sg==
+Message-ID: <427999f6-114f-e59c-631e-ab2a5f6bfe4c@ueber.net>
+Date: Fri, 10 Dec 2021 20:09:08 +0100
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101
+ Thunderbird/91.4.0
+Content-Language: nl
+To: mechiel@ueber.net
+From: Mechiel Lukkien <mechiel@ueber.net>
+Subject: test
+Content-Type: text/plain; charset=UTF-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+test
+`, "\n", "\r\n")
+
+	resolver := dns.MockResolver{
+		TXT: map[string][]string{
+			"koriander._domainkey.ueber.net.": {"v=DKIM1; k=rsa; s=email; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy3Z9ffZe8gUTJrdGuKj6IwEembmKYpp0jMa8uhudErcI4gFVUaFiiRWxc4jP/XR9NAEv3XwHm+CVcHu+L/n6VWt6g59U7vHXQicMfKGmEp2VplsgojNy/Y5X9HdVYM0azsI47NcJCDW9UVfeOHdOSgFME4F8dNtUKC4KTB2d1pqj/yixz+V8Sv8xkEyPfSRHcNXIw0LvelqJ1MRfN3hO/3uQSVrPYYk4SyV0b6wfnkQs28fpiIpGQvzlGI5WkrdOQT5k4YHaEvZDLNdwiMeVZOEL7dDoFs2mQsovm+tH0StUAZTnr61NLVFfD5V6Ip1V9zVtspPHvYSuOWwyArFZ9QIDAQAB"},
+		},
+	}
+
+	results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false)
+	if err != nil {
+		t.Fatalf("dkim verify: %v", err)
+	}
+	if len(results) != 1 || results[0].Status != StatusPass {
+		t.Fatalf("verify: unexpected results %v", results)
+	}
+}
+
+func TestVerifyEd25519(t *testing.T) {
+	// ../rfc/8463:287
+	message := strings.ReplaceAll(`DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed;
+ d=football.example.com; i=@football.example.com;
+ q=dns/txt; s=brisbane; t=1528637909; h=from : to :
+ subject : date : message-id : from : subject : date;
+ bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=;
+ b=/gCrinpcQOoIfuHNQIbq4pgh9kyIK3AQUdt9OdqQehSwhEIug4D11Bus
+ Fa3bT3FY5OsU7ZbnKELq+eXdp1Q1Dw==
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+ d=football.example.com; i=@football.example.com;
+ q=dns/txt; s=test; t=1528637909; h=from : to : subject :
+ date : message-id : from : subject : date;
+ bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=;
+ b=F45dVWDfMbQDGHJFlXUNB2HKfbCeLRyhDXgFpEL8GwpsRe0IeIixNTe3
+ DhCVlUrSjV4BwcVcOF6+FF3Zo9Rpo1tFOeS9mPYQTnGdaSGsgeefOsk2Jz
+ dA+L10TeYt9BgDfQNZtKdN1WO//KgIqXP7OdEFE4LjFYNcUxZQ4FADY+8=
+From: Joe SixPack <joe@football.example.com>
+To: Suzie Q <suzie@shopping.example.net>
+Subject: Is dinner ready?
+Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT)
+Message-ID: <20030712040037.46341.5F8J@football.example.com>
+
+Hi.
+
+We lost the game.  Are you hungry yet?
+
+Joe.
+
+`, "\n", "\r\n")
+
+	resolver := dns.MockResolver{
+		TXT: map[string][]string{
+			"brisbane._domainkey.football.example.com.": {"v=DKIM1; k=ed25519; p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo="},
+			"test._domainkey.football.example.com.":     {"v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDkHlOQoBTzWRiGs5V6NpP3idY6Wk08a5qhdR6wy5bdOKb2jLQiY/J16JYi0Qvx/byYzCNb3W91y3FutACDfzwQ/BC/e/8uBsCR+yz1Lxj+PL6lHvqMKrM3rG4hstT5QjvHO9PzoxZyVYLzBfO2EeC3Ip3G+2kryOTIKT+l/K4w3QIDAQAB"},
+		},
+	}
+
+	results, err := Verify(context.Background(), resolver, false, policyOK, strings.NewReader(message), false)
+	if err != nil {
+		t.Fatalf("dkim verify: %v", err)
+	}
+	if len(results) != 2 || results[0].Status != StatusPass || results[1].Status != StatusPass {
+		t.Fatalf("verify: unexpected results %#v", results)
+	}
+}
+
+func TestSign(t *testing.T) {
+	message := strings.ReplaceAll(`Message-ID: <427999f6-114f-e59c-631e-ab2a5f6bfe4c@ueber.net>
+Date: Fri, 10 Dec 2021 20:09:08 +0100
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101
+ Thunderbird/91.4.0
+Content-Language: nl
+To: mechiel@ueber.net
+From: Mechiel Lukkien <mechiel@ueber.net>
+Subject: test
+ test
+Content-Type: text/plain; charset=UTF-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+test
+`, "\n", "\r\n")
+
+	rsaKey := getRSAKey(t)
+	ed25519Key := ed25519.NewKeyFromSeed(make([]byte, 32))
+
+	selrsa := config.Selector{
+		HashEffective:    "sha256",
+		Key:              rsaKey,
+		HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
+		Domain:           dns.Domain{ASCII: "testrsa"},
+	}
+
+	// Now with sha1 and relaxed canonicalization.
+	selrsa2 := config.Selector{
+		HashEffective:    "sha1",
+		Key:              rsaKey,
+		HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
+		Domain:           dns.Domain{ASCII: "testrsa2"},
+	}
+	selrsa2.Canonicalization.HeaderRelaxed = true
+	selrsa2.Canonicalization.BodyRelaxed = true
+
+	// Ed25519 key.
+	seled25519 := config.Selector{
+		HashEffective:    "sha256",
+		Key:              ed25519Key,
+		HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
+		Domain:           dns.Domain{ASCII: "tested25519"},
+	}
+	// Again ed25519, but without sealing headers. Use sha256 again, for reusing the body hash from the previous dkim-signature.
+	seled25519b := config.Selector{
+		HashEffective:    "sha256",
+		Key:              ed25519Key,
+		HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,Subject,Date", ","),
+		DontSealHeaders:  true,
+		Domain:           dns.Domain{ASCII: "tested25519b"},
+	}
+	dkimConf := config.DKIM{
+		Selectors: map[string]config.Selector{
+			"testrsa":      selrsa,
+			"testrsa2":     selrsa2,
+			"tested25519":  seled25519,
+			"tested25519b": seled25519b,
+		},
+		Sign: []string{"testrsa", "testrsa2", "tested25519", "tested25519b"},
+	}
+
+	ctx := context.Background()
+	headers, err := Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message))
+	if err != nil {
+		t.Fatalf("sign: %v", err)
+	}
+
+	makeRecord := func(k string, publicKey any) string {
+		tr := &Record{
+			Version:   "DKIM1",
+			Key:       k,
+			PublicKey: publicKey,
+			Flags:     []string{"s"},
+		}
+		txt, err := tr.Record()
+		if err != nil {
+			t.Fatalf("making dns txt record: %s", err)
+		}
+		//log.Infof("txt record: %s", txt)
+		return txt
+	}
+
+	resolver := dns.MockResolver{
+		TXT: map[string][]string{
+			"testrsa._domainkey.mox.example.":      {makeRecord("rsa", rsaKey.Public())},
+			"testrsa2._domainkey.mox.example.":     {makeRecord("rsa", rsaKey.Public())},
+			"tested25519._domainkey.mox.example.":  {makeRecord("ed25519", ed25519Key.Public())},
+			"tested25519b._domainkey.mox.example.": {makeRecord("ed25519", ed25519Key.Public())},
+		},
+	}
+
+	nmsg := headers + message
+
+	results, err := Verify(ctx, resolver, false, policyOK, strings.NewReader(nmsg), false)
+	if err != nil {
+		t.Fatalf("verify: %s", err)
+	}
+	if len(results) != 4 || results[0].Status != StatusPass || results[1].Status != StatusPass || results[2].Status != StatusPass || results[3].Status != StatusPass {
+		t.Fatalf("verify: unexpected results %v\nheaders:\n%s", results, headers)
+	}
+	//log.Infof("headers:%s", headers)
+	//log.Infof("nmsg\n%s", nmsg)
+
+	// Multiple From headers.
+	_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest"))
+	if !errors.Is(err, ErrFrom) {
+		t.Fatalf("sign, got err %v, expected ErrFrom", err)
+	}
+
+	// No From header.
+	_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest"))
+	if !errors.Is(err, ErrFrom) {
+		t.Fatalf("sign, got err %v, expected ErrFrom", err)
+	}
+
+	// Malformed headers.
+	_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest"))
+	if !errors.Is(err, ErrHeaderMalformed) {
+		t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
+	}
+	_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest"))
+	if !errors.Is(err, ErrHeaderMalformed) {
+		t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
+	}
+	_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest"))
+	if !errors.Is(err, ErrHeaderMalformed) {
+		t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
+	}
+	_, err = Sign(ctx, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:<mjl@mox.example>"))
+	if !errors.Is(err, ErrHeaderMalformed) {
+		t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
+	}
+}
+
+func TestVerify(t *testing.T) {
+	// We do many Verify calls, each time starting out with a valid configuration, then
+	// we modify one thing to trigger an error, which we check for.
+
+	const message = `From: <mjl@mox.example>
+To: <other@mox.example>
+Subject: test
+Date: Fri, 10 Dec 2021 20:09:08 +0100
+Message-ID: <test@mox.example>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+test
+`
+
+	key := ed25519.NewKeyFromSeed(make([]byte, 32))
+	var resolver dns.MockResolver
+	var record *Record
+	var recordTxt string
+	var msg string
+	var sel config.Selector
+	var dkimConf config.DKIM
+	var policy func(*Sig) error
+	var signed bool
+	var signDomain dns.Domain
+
+	prepare := func() {
+		t.Helper()
+
+		policy = DefaultPolicy
+		signDomain = dns.Domain{ASCII: "mox.example"}
+
+		record = &Record{
+			Version:   "DKIM1",
+			Key:       "ed25519",
+			PublicKey: key.Public(),
+			Flags:     []string{"s"},
+		}
+
+		txt, err := record.Record()
+		if err != nil {
+			t.Fatalf("making dns txt record: %s", err)
+		}
+		recordTxt = txt
+
+		resolver = dns.MockResolver{
+			TXT: map[string][]string{
+				"test._domainkey.mox.example.": {txt},
+			},
+		}
+
+		sel = config.Selector{
+			HashEffective:    "sha256",
+			Key:              key,
+			HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
+			Domain:           dns.Domain{ASCII: "test"},
+		}
+		dkimConf = config.DKIM{
+			Selectors: map[string]config.Selector{
+				"test": sel,
+			},
+			Sign: []string{"test"},
+		}
+
+		msg = message
+		signed = false
+	}
+
+	sign := func() {
+		t.Helper()
+
+		msg = strings.ReplaceAll(msg, "\n", "\r\n")
+
+		headers, err := Sign(context.Background(), "mjl", signDomain, dkimConf, false, strings.NewReader(msg))
+		if err != nil {
+			t.Fatalf("sign: %v", err)
+		}
+		msg = headers + msg
+		signed = true
+	}
+
+	test := func(expErr error, expStatus Status, expResultErr error, mod func()) {
+		t.Helper()
+
+		prepare()
+		mod()
+		if !signed {
+			sign()
+		}
+
+		results, err := Verify(context.Background(), resolver, true, policy, strings.NewReader(msg), false)
+		if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
+			t.Fatalf("got verify error %v, expected %v", err, expErr)
+		}
+		if expStatus != "" && (len(results) == 0 || results[0].Status != expStatus) {
+			var status Status
+			if len(results) > 0 {
+				status = results[0].Status
+			}
+			t.Fatalf("got status %q, expected %q", status, expStatus)
+		}
+		var resultErr error
+		if len(results) > 0 {
+			resultErr = results[0].Err
+		}
+		if (resultErr == nil) != (expResultErr == nil) || resultErr != nil && !errors.Is(resultErr, expResultErr) {
+			t.Fatalf("got result error %v, expected %v", resultErr, expResultErr)
+		}
+	}
+
+	test(nil, StatusPass, nil, func() {})
+
+	// Cannot parse message, so not much more to do.
+	test(ErrHeaderMalformed, "", nil, func() {
+		sign()
+		msg = ":\r\n\r\n" // Empty header key.
+	})
+
+	// From Lookup.
+	// No DKIM record. ../rfc/6376:2608
+	test(nil, StatusPermerror, ErrNoRecord, func() {
+		resolver.TXT = nil
+	})
+	// DNS request is failing temporarily.
+	test(nil, StatusTemperror, ErrDNS, func() {
+		resolver.Fail = map[dns.Mockreq]struct{}{
+			{Type: "txt", Name: "test._domainkey.mox.example."}: {},
+		}
+	})
+	// Claims to be DKIM through v=, but cannot be parsed. ../rfc/6376:2621
+	test(nil, StatusPermerror, ErrSyntax, func() {
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {"v=DKIM1; bogus"},
+		}
+	})
+	// Not a DKIM record. ../rfc/6376:2621
+	test(nil, StatusTemperror, ErrSyntax, func() {
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {"bogus"},
+		}
+	})
+	// Multiple dkim records. ../rfc/6376:1609
+	test(nil, StatusTemperror, ErrMultipleRecords, func() {
+		resolver.TXT["test._domainkey.mox.example."] = []string{recordTxt, recordTxt}
+	})
+
+	// Invalid DKIM-Signature header. ../rfc/6376:2503
+	test(nil, StatusPermerror, errSigMissingTag, func() {
+		msg = strings.ReplaceAll("DKIM-Signature: v=1\n"+msg, "\n", "\r\n")
+		signed = true
+	})
+
+	// Signature has valid syntax, but parameters aren't acceptable.
+	// "From" not signed.  ../rfc/6376:2546
+	test(nil, StatusPermerror, ErrFrom, func() {
+		sign()
+		// Remove "from" from signed headers (h=).
+		msg = strings.ReplaceAll(msg, ":From:", ":")
+		msg = strings.ReplaceAll(msg, "=From:", "=")
+	})
+	// todo: check expired signatures with StatusPermerror and ErrSigExpired. ../rfc/6376:2550
+	// Domain in signature is higher-level than organizational domain. ../rfc/6376:2554
+	test(nil, StatusPermerror, ErrTLD, func() {
+		// Pretend to sign as .com
+		msg = strings.ReplaceAll(msg, "From: <mjl@mox.example>\n", "From: <mjl@com>\n")
+		signDomain = dns.Domain{ASCII: "com"}
+		resolver.TXT = map[string][]string{
+			"test._domainkey.com.": {recordTxt},
+		}
+	})
+	// Unknown hash algorithm.
+	test(nil, StatusPermerror, ErrHashAlgorithmUnknown, func() {
+		sign()
+		msg = strings.ReplaceAll(msg, "sha256", "sha257")
+	})
+	// Unknown canonicalization.
+	test(nil, StatusPermerror, ErrCanonicalizationUnknown, func() {
+		sel.Canonicalization.HeaderRelaxed = true
+		sel.Canonicalization.BodyRelaxed = true
+		dkimConf.Selectors = map[string]config.Selector{
+			"test": sel,
+		}
+
+		sign()
+		msg = strings.ReplaceAll(msg, "relaxed/relaxed", "bogus/bogus")
+	})
+	// Query methods without dns/txt. ../rfc/6376:1268
+	test(nil, StatusPermerror, ErrQueryMethod, func() {
+		sign()
+		msg = strings.ReplaceAll(msg, "DKIM-Signature: ", "DKIM-Signature: q=other;")
+	})
+
+	// Unacceptable through policy. ../rfc/6376:2560
+	test(nil, StatusPolicy, ErrPolicy, func() {
+		sign()
+		msg = strings.ReplaceAll(msg, "DKIM-Signature: ", "DKIM-Signature: l=1;")
+	})
+	// Hash algorithm not allowed by DNS record. ../rfc/6376:2639
+	test(nil, StatusPermerror, ErrHashAlgNotAllowed, func() {
+		recordTxt += ";h=sha1"
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {recordTxt},
+		}
+	})
+	// Signature algorithm mismatch. ../rfc/6376:2651
+	test(nil, StatusPermerror, ErrSigAlgMismatch, func() {
+		record.PublicKey = getRSAKey(t).Public()
+		record.Key = "rsa"
+		txt, err := record.Record()
+		if err != nil {
+			t.Fatalf("making dns txt record: %s", err)
+		}
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {txt},
+		}
+	})
+	// Empty public key means revoked key. ../rfc/6376:2645
+	test(nil, StatusPermerror, ErrKeyRevoked, func() {
+		record.PublicKey = nil
+		txt, err := record.Record()
+		if err != nil {
+			t.Fatalf("making dns txt record: %s", err)
+		}
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {txt},
+		}
+	})
+	// We refuse rsa keys smaller than 1024 bits.
+	test(nil, StatusPermerror, ErrWeakKey, func() {
+		key := getWeakRSAKey(t)
+		record.Key = "rsa"
+		record.PublicKey = key.Public()
+		txt, err := record.Record()
+		if err != nil {
+			t.Fatalf("making dns txt record: %s", err)
+		}
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {txt},
+		}
+		sel.Key = key
+		dkimConf.Selectors = map[string]config.Selector{
+			"test": sel,
+		}
+	})
+	// Key not allowed for email by DNS record. ../rfc/6376:1541
+	test(nil, StatusPermerror, ErrKeyNotForEmail, func() {
+		recordTxt += ";s=other"
+		resolver.TXT = map[string][]string{
+			"test._domainkey.mox.example.": {recordTxt},
+		}
+	})
+	// todo: Record has flag "s" but identity does not have exact domain match. Cannot currently easily implement this test because Sign() always uses the same domain. ../rfc/6376:1575
+	// Wrong signature, different datahash, and thus signature.
+	test(nil, StatusFail, ErrSigVerify, func() {
+		sign()
+		msg = strings.ReplaceAll(msg, "Subject: test\r\n", "Subject: modified header\r\n")
+	})
+	// Signature is correct for bodyhash, but the body has changed.
+	test(nil, StatusFail, ErrBodyhashMismatch, func() {
+		sign()
+		msg = strings.ReplaceAll(msg, "\r\ntest\r\n", "\r\nmodified body\r\n")
+	})
+
+	// Check that last-occurring header field is used.
+	test(nil, StatusFail, ErrSigVerify, func() {
+		sel.DontSealHeaders = true
+		dkimConf.Selectors = map[string]config.Selector{
+			"test": sel,
+		}
+		sign()
+		msg = strings.ReplaceAll(msg, "\r\n\r\n", "\r\nsubject: another\r\n\r\n")
+	})
+	test(nil, StatusPass, nil, func() {
+		sel.DontSealHeaders = true
+		dkimConf.Selectors = map[string]config.Selector{
+			"test": sel,
+		}
+		sign()
+		msg = "subject: another\r\n" + msg
+	})
+}
+
+func TestBodyHash(t *testing.T) {
+	simpleGot, err := bodyHash(crypto.SHA256.New(), true, bufio.NewReader(strings.NewReader("")))
+	if err != nil {
+		t.Fatalf("body hash, simple, empty string: %s", err)
+	}
+	simpleWant := base64Decode("frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN/XKdLCPjaYaY=")
+	if !bytes.Equal(simpleGot, simpleWant) {
+		t.Fatalf("simple body hash for empty string, got %s, expected %s", base64Encode(simpleGot), base64Encode(simpleWant))
+	}
+
+	relaxedGot, err := bodyHash(crypto.SHA256.New(), false, bufio.NewReader(strings.NewReader("")))
+	if err != nil {
+		t.Fatalf("body hash, relaxed, empty string: %s", err)
+	}
+	relaxedWant := base64Decode("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=")
+	if !bytes.Equal(relaxedGot, relaxedWant) {
+		t.Fatalf("relaxed body hash for empty string, got %s, expected %s", base64Encode(relaxedGot), base64Encode(relaxedWant))
+	}
+
+	compare := func(a, b []byte) {
+		t.Helper()
+		if !bytes.Equal(a, b) {
+			t.Fatalf("hash not equal")
+		}
+	}
+
+	// NOTE: the trailing space in the strings below are part of the test for canonicalization.
+
+	// ../rfc/6376:936
+	exampleIn := strings.ReplaceAll(` c
+d 	 e
+
+
+`, "\n", "\r\n")
+	relaxedOut := strings.ReplaceAll(` c
+d e
+`, "\n", "\r\n")
+	relaxedBh, err := bodyHash(crypto.SHA256.New(), false, bufio.NewReader(strings.NewReader(exampleIn)))
+	if err != nil {
+		t.Fatalf("bodyhash: %s", err)
+	}
+	relaxedOutHash := sha256.Sum256([]byte(relaxedOut))
+	compare(relaxedBh, relaxedOutHash[:])
+
+	simpleOut := strings.ReplaceAll(` c
+d 	 e
+`, "\n", "\r\n")
+	simpleBh, err := bodyHash(crypto.SHA256.New(), true, bufio.NewReader(strings.NewReader(exampleIn)))
+	if err != nil {
+		t.Fatalf("bodyhash: %s", err)
+	}
+	simpleOutHash := sha256.Sum256([]byte(simpleOut))
+	compare(simpleBh, simpleOutHash[:])
+
+	// ../rfc/8463:343
+	relaxedBody := strings.ReplaceAll(`Hi.
+
+We lost the game.  Are you hungry yet?
+
+Joe.
+
+`, "\n", "\r\n")
+	relaxedGot, err = bodyHash(crypto.SHA256.New(), false, bufio.NewReader(strings.NewReader(relaxedBody)))
+	if err != nil {
+		t.Fatalf("body hash, relaxed, ed25519 example: %s", err)
+	}
+	relaxedWant = base64Decode("2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=")
+	if !bytes.Equal(relaxedGot, relaxedWant) {
+		t.Fatalf("relaxed body hash for ed25519 example, got %s, expected %s", base64Encode(relaxedGot), base64Encode(relaxedWant))
+	}
+}
+
+func base64Decode(s string) []byte {
+	buf, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		panic(err)
+	}
+	return buf
+}
+
+func base64Encode(buf []byte) string {
+	return base64.StdEncoding.EncodeToString(buf)
+}

dkim/fuzz_test.go

@@ -0,0 +1,25 @@
+package dkim
+
+import (
+	"testing"
+)
+
+func FuzzParseSignature(f *testing.F) {
+	f.Add([]byte(""))
+	f.Fuzz(func(t *testing.T, buf []byte) {
+		parseSignature(buf, false)
+	})
+}
+
+func FuzzParseRecord(f *testing.F) {
+	f.Add("")
+	f.Add("v=DKIM1; p=bad")
+	f.Fuzz(func(t *testing.T, s string) {
+		r, _, err := ParseRecord(s)
+		if err == nil {
+			if _, err := r.Record(); err != nil {
+				t.Errorf("r.Record() for parsed record %s, %#v: %s", s, r, err)
+			}
+		}
+	})
+}

dkim/parser.go

@@ -0,0 +1,474 @@
+package dkim
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/smtp"
+)
+
+type parseErr string
+
+func (e parseErr) Error() string {
+	return string(e)
+}
+
+var _ error = parseErr("")
+
+type parser struct {
+	s        string
+	o        int    // Offset into s.
+	tracked  string // All data consumed, except when "drop" is true. To be set by caller when parsing the value for "b=".
+	drop     bool
+	smtputf8 bool // If set, allow characters > 0x7f.
+}
+
+func (p *parser) xerrorf(format string, args ...any) {
+	msg := fmt.Sprintf(format, args...)
+	if p.o < len(p.s) {
+		msg = fmt.Sprintf("%s (leftover %q)", msg, p.s[p.o:])
+	}
+	panic(parseErr(msg))
+}
+
+func (p *parser) track(s string) {
+	if !p.drop {
+		p.tracked += s
+	}
+}
+
+func (p *parser) hasPrefix(s string) bool {
+	return strings.HasPrefix(p.s[p.o:], s)
+}
+
+func (p *parser) xtaken(n int) string {
+	r := p.s[p.o : p.o+n]
+	p.o += n
+	p.track(r)
+	return r
+}
+
+func (p *parser) xtakefn(fn func(c rune, i int) bool) string {
+	for i, c := range p.s[p.o:] {
+		if !fn(c, i) {
+			return p.xtaken(i)
+		}
+	}
+	return p.xtaken(len(p.s) - p.o)
+}
+
+func (p *parser) empty() bool {
+	return p.o >= len(p.s)
+}
+
+func (p *parser) xnonempty() {
+	if p.o >= len(p.s) {
+		p.xerrorf("expected at least 1 more char")
+	}
+}
+
+func (p *parser) xtakefn1(fn func(c rune, i int) bool) string {
+	p.xnonempty()
+	for i, c := range p.s[p.o:] {
+		if !fn(c, i) {
+			if i == 0 {
+				p.xerrorf("expected at least 1 char")
+			}
+			return p.xtaken(i)
+		}
+	}
+	return p.xtaken(len(p.s) - p.o)
+}
+
+func (p *parser) wsp() {
+	p.xtakefn(func(c rune, i int) bool {
+		return c == ' ' || c == '\t'
+	})
+}
+
+func (p *parser) fws() {
+	p.wsp()
+	if p.hasPrefix("\r\n ") || p.hasPrefix("\r\n\t") {
+		p.xtaken(3)
+		p.wsp()
+	}
+}
+
+// peekfws returns whether remaining text starts with s, optionally prefix with fws.
+func (p *parser) peekfws(s string) bool {
+	o := p.o
+	p.fws()
+	r := p.hasPrefix(s)
+	p.o = o
+	return r
+}
+
+func (p *parser) xtake(s string) string {
+	if !strings.HasPrefix(p.s[p.o:], s) {
+		p.xerrorf("expected %q", s)
+	}
+	return p.xtaken(len(s))
+}
+
+func (p *parser) take(s string) bool {
+	if strings.HasPrefix(p.s[p.o:], s) {
+		p.o += len(s)
+		p.track(s)
+		return true
+	}
+	return false
+}
+
+// ../rfc/6376:657
+func (p *parser) xtagName() string {
+	return p.xtakefn1(func(c rune, i int) bool {
+		return isalpha(c) || i > 0 && (isdigit(c) || c == '_')
+	})
+}
+
+func (p *parser) xalgorithm() (string, string) {
+	// ../rfc/6376:1046
+	xtagx := func(c rune, i int) bool {
+		return isalpha(c) || i > 0 && isdigit(c)
+	}
+	algk := p.xtakefn1(xtagx)
+	p.xtake("-")
+	algv := p.xtakefn1(xtagx)
+	return algk, algv
+}
+
+// fws in value is ignored. empty/no base64 characters is valid.
+// ../rfc/6376:1021
+// ../rfc/6376:1076
+func (p *parser) xbase64() []byte {
+	s := ""
+	p.xtakefn(func(c rune, i int) bool {
+		if isalphadigit(c) || c == '+' || c == '/' || c == '=' {
+			s += string(c)
+			return true
+		}
+		if c == ' ' || c == '\t' {
+			return true
+		}
+		rem := p.s[p.o+i:]
+		if strings.HasPrefix(rem, "\r\n ") || strings.HasPrefix(rem, "\r\n\t") {
+			return true
+		}
+		if (strings.HasPrefix(rem, "\n ") || strings.HasPrefix(rem, "\n\t")) && p.o+i-1 > 0 && p.s[p.o+i-1] == '\r' {
+			return true
+		}
+		return false
+	})
+	buf, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		p.xerrorf("decoding base64: %v", err)
+	}
+	return buf
+}
+
+// parses canonicalization in original case.
+func (p *parser) xcanonical() string {
+	// ../rfc/6376:1100
+	s := p.xhyphenatedWord()
+	if p.take("/") {
+		return s + "/" + p.xhyphenatedWord()
+	}
+	return s
+}
+
+func (p *parser) xdomain() dns.Domain {
+	subdomain := func(c rune, i int) bool {
+		// domain names must always be a-labels, ../rfc/6376:1115 ../rfc/6376:1187 ../rfc/6376:1303
+		// todo: add a "lax" mode where underscore is allowed if this is a selector? seen in the wild, but invalid: ../rfc/6376:581 ../rfc/5321:2303
+		return isalphadigit(c) || (i > 0 && c == '-' && p.o+1 < len(p.s))
+	}
+	s := p.xtakefn1(subdomain)
+	for p.hasPrefix(".") {
+		s += p.xtake(".") + p.xtakefn1(subdomain)
+	}
+	d, err := dns.ParseDomain(s)
+	if err != nil {
+		p.xerrorf("parsing domain %q: %s", s, err)
+	}
+	return d
+}
+
+func (p *parser) xhdrName() string {
+	// ../rfc/6376:473
+	// ../rfc/5322:1689
+	// BNF for hdr-name (field-name) allows ";", but DKIM disallows unencoded semicolons. ../rfc/6376:643
+	return p.xtakefn1(func(c rune, i int) bool {
+		return c > ' ' && c < 0x7f && c != ':' && c != ';'
+	})
+}
+
+func (p *parser) xsignedHeaderFields() []string {
+	// ../rfc/6376:1157
+	l := []string{p.xhdrName()}
+	for p.peekfws(":") {
+		p.fws()
+		p.xtake(":")
+		p.fws()
+		l = append(l, p.xhdrName())
+	}
+	return l
+}
+
+func (p *parser) xauid() Identity {
+	// ../rfc/6376:1192
+	// Localpart is optional.
+	if p.take("@") {
+		return Identity{Domain: p.xdomain()}
+	}
+	lp := p.xlocalpart()
+	p.xtake("@")
+	dom := p.xdomain()
+	return Identity{&lp, dom}
+}
+
+// todo: reduce duplication between implementations: ../smtp/address.go:/xlocalpart ../dkim/parser.go:/xlocalpart ../smtpserver/parse.go:/xlocalpart
+func (p *parser) xlocalpart() smtp.Localpart {
+	// ../rfc/6376:434
+	// ../rfc/5321:2316
+	var s string
+	if p.hasPrefix(`"`) {
+		s = p.xquotedString()
+	} else {
+		s = p.xatom()
+		for p.take(".") {
+			s += "." + p.xatom()
+		}
+	}
+	// todo: have a strict parser that only allows the actual max of 64 bytes. some services have large localparts because of generated (bounce) addresses.
+	if len(s) > 128 {
+		// ../rfc/5321:3486
+		p.xerrorf("localpart longer than 64 octets")
+	}
+	return smtp.Localpart(s)
+}
+
+func (p *parser) xquotedString() string {
+	p.xtake(`"`)
+	var s string
+	var esc bool
+	for {
+		c := p.xchar()
+		if esc {
+			if c >= ' ' && c < 0x7f {
+				s += string(c)
+				esc = false
+				continue
+			}
+			p.xerrorf("invalid localpart, bad escaped char %c", c)
+		}
+		if c == '\\' {
+			esc = true
+			continue
+		}
+		if c == '"' {
+			return s
+		}
+		if c >= ' ' && c < 0x7f && c != '\\' && c != '"' || (c > 0x7f && p.smtputf8) {
+			s += string(c)
+			continue
+		}
+		p.xerrorf("invalid localpart, invalid character %c", c)
+	}
+}
+
+func (p *parser) xchar() rune {
+	// We are careful to track invalid utf-8 properly.
+	if p.empty() {
+		p.xerrorf("need another character")
+	}
+	var r rune
+	var o int
+	for i, c := range p.s[p.o:] {
+		if i > 0 {
+			o = i
+			break
+		}
+		r = c
+	}
+	if o == 0 {
+		p.track(p.s[p.o:])
+		p.o = len(p.s)
+	} else {
+		p.track(p.s[p.o : p.o+o])
+		p.o += o
+	}
+	return r
+}
+
+func (p *parser) xatom() string {
+	return p.xtakefn1(func(c rune, i int) bool {
+		switch c {
+		case '!', '#', '$', '%', '&', '\'', '*', '+', '-', '/', '=', '?', '^', '_', '`', '{', '|', '}', '~':
+			return true
+		}
+		return isalphadigit(c) || (c > 0x7f && p.smtputf8)
+	})
+}
+
+func (p *parser) xbodyLength() int64 {
+	// ../rfc/6376:1265
+	return p.xnumber(76)
+}
+
+func (p *parser) xnumber(maxdigits int) int64 {
+	o := -1
+	for i, c := range p.s[p.o:] {
+		if c >= '0' && c <= '9' {
+			o = i
+		} else {
+			break
+		}
+	}
+	if o == -1 {
+		p.xerrorf("expected digits")
+	}
+	if o+1 > maxdigits {
+		p.xerrorf("too many digits")
+	}
+	v, err := strconv.ParseInt(p.xtaken(o+1), 10, 64)
+	if err != nil {
+		p.xerrorf("parsing digits: %s", err)
+	}
+	return v
+}
+
+func (p *parser) xqueryMethods() []string {
+	// ../rfc/6376:1285
+	l := []string{p.xqtagmethod()}
+	for p.peekfws(":") {
+		p.fws()
+		p.xtake(":")
+		l = append(l, p.xqtagmethod())
+	}
+	return l
+}
+
+func (p *parser) xqtagmethod() string {
+	// ../rfc/6376:1295 ../rfc/6376-eid4810
+	s := p.xhyphenatedWord()
+	// ABNF production "x-sig-q-tag-args" should probably just have been
+	// "hyphenated-word". As qp-hdr-value, it will consume ":". A similar problem does
+	// not occur for "z" because it is also "|"-delimited. We work around the potential
+	// issue by parsing "dns/txt" explicitly.
+	rem := p.s[p.o:]
+	if strings.EqualFold(s, "dns") && len(rem) >= len("/txt") && strings.EqualFold(rem[:len("/txt")], "/txt") {
+		s += p.xtaken(4)
+	} else if p.take("/") {
+		s += "/" + p.xqp(true, true)
+	}
+	return s
+}
+
+func isalpha(c rune) bool {
+	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'
+}
+
+func isdigit(c rune) bool {
+	return c >= '0' && c <= '9'
+}
+
+func isalphadigit(c rune) bool {
+	return isalpha(c) || isdigit(c)
+}
+
+// ../rfc/6376:469
+func (p *parser) xhyphenatedWord() string {
+	return p.xtakefn1(func(c rune, i int) bool {
+		return isalpha(c) || i > 0 && isdigit(c) || i > 0 && c == '-' && p.o+i+1 < len(p.s) && isalphadigit(rune(p.s[p.o+i+1]))
+	})
+}
+
+// ../rfc/6376:474
+func (p *parser) xqphdrvalue() string {
+	return p.xqp(true, false)
+}
+
+func (p *parser) xqpSection() string {
+	return p.xqp(false, false)
+}
+
+// dkim-quoted-printable (pipeEncoded true) or qp-section.
+//
+// It is described in terms of (lots of) modifications to MIME quoted-printable,
+// but it may be simpler to just ignore that reference.
+func (p *parser) xqp(pipeEncoded, colonEncoded bool) string {
+	// ../rfc/6376:494 ../rfc/2045:1260
+
+	hex := func(c byte) rune {
+		if c >= '0' && c <= '9' {
+			return rune(c - '0')
+		}
+		return rune(10 + c - 'A')
+	}
+
+	s := ""
+	for !p.empty() {
+		p.fws()
+		if pipeEncoded && p.hasPrefix("|") {
+			break
+		}
+		if colonEncoded && p.hasPrefix(":") {
+			break
+		}
+		if p.hasPrefix("=") {
+			p.xtake("=")
+			// note: \r\n before the full hex-octet has been encountered in the wild. Could be
+			// a sender just wrapping their headers after escaping, or not escaping an "=". We
+			// currently don't compensate for it.
+			h := p.xtakefn(func(c rune, i int) bool {
+				return i < 2 && (c >= '0' && c <= '9' || c >= 'A' && c <= 'Z')
+			})
+			if len(h) != 2 {
+				p.xerrorf("expected qp-hdr-value")
+			}
+			c := (hex(h[0]) << 4) | hex(h[1])
+			s += string(c)
+			continue
+		}
+		x := p.xtakefn(func(c rune, i int) bool {
+			return c > ' ' && c < 0x7f && c != ';' && c != '=' && !(pipeEncoded && c == '|')
+		})
+		if x == "" {
+			break
+		}
+		s += x
+	}
+	return s
+}
+
+func (p *parser) xselector() dns.Domain {
+	return p.xdomain()
+}
+
+func (p *parser) xtimestamp() int64 {
+	// ../rfc/6376:1325 ../rfc/6376:1358
+	return p.xnumber(12)
+}
+
+func (p *parser) xcopiedHeaderFields() []string {
+	// ../rfc/6376:1384
+	l := []string{p.xztagcopy()}
+	for p.hasPrefix("|") {
+		p.xtake("|")
+		p.fws()
+		l = append(l, p.xztagcopy())
+	}
+	return l
+}
+
+func (p *parser) xztagcopy() string {
+	// ../rfc/6376:1386
+	f := p.xhdrName()
+	p.fws()
+	p.xtake(":")
+	v := p.xqphdrvalue()
+	return f + ":" + v
+}

dkim/policy.go

@@ -0,0 +1,49 @@
+package dkim
+
+import (
+	"fmt"
+	"strings"
+)
+
+// DefaultPolicy is the default DKIM policy.
+//
+// Signatures with a length restriction are rejected because it is hard to decide
+// how many signed bytes should be required (none? at least half? all except
+// max N bytes?). Also, it isn't likely email applications (MUAs) will be
+// displaying the signed vs unsigned (partial) content differently, mostly
+// because the encoded data is signed. E.g.  half a base64 image could be
+// signed, and the rest unsigned.
+//
+// Signatures without Subject field are rejected. The From header field is
+// always required and does not need to be checked in the policy.
+// Other signatures are accepted.
+func DefaultPolicy(sig *Sig) error {
+	// ../rfc/6376:2088
+	// ../rfc/6376:2307
+	// ../rfc/6376:2706
+	// ../rfc/6376:1558
+	if sig.Length >= 0 {
+		return fmt.Errorf("l= for length not acceptable")
+	}
+
+	// ../rfc/6376:2139
+	// We require at least the following headers: From, Subject.
+	// You would expect To, Cc and Message-ID to also always be present.
+	// Microsoft appears to leave out To.
+	// Yahoo appears to leave out Message-ID.
+	// Multiple leave out Cc and other address headers.
+	// At least one newsletter did not sign Date.
+	var subject bool
+	for _, h := range sig.SignedHeaders {
+		subject = subject || strings.EqualFold(h, "subject")
+	}
+	var missing []string
+	if !subject {
+		missing = append(missing, "subject")
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("required header fields missing from signature: %s", strings.Join(missing, ", "))
+	}
+
+	return nil
+}

dkim/sig.go

@@ -0,0 +1,353 @@
+package dkim
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/smtp"
+)
+
+// Sig is a DKIM-Signature header.
+//
+// String values must be compared case insensitively.
+type Sig struct {
+	// Required fields.
+	Version       int        // Version, 1. Field "v". Always the first field.
+	AlgorithmSign string     // "rsa" or "ed25519". Field "a".
+	AlgorithmHash string     // "sha256" or the deprecated "sha1" (deprecated). Field "a".
+	Signature     []byte     // Field "b".
+	BodyHash      []byte     // Field "bh".
+	Domain        dns.Domain // Field "d".
+	SignedHeaders []string   // Duplicates are meaningful. Field "h".
+	Selector      dns.Domain // Selector, for looking DNS TXT record at <s>._domainkey.<domain>. Field "s".
+
+	// Optional fields.
+	// Canonicalization is the transformation of header and/or body before hashing. The
+	// value is in original case, but must be compared case-insensitively. Normally two
+	// slash-separated values: header canonicalization and body canonicalization. But
+	// the "simple" means "simple/simple" and "relaxed" means "relaxed/simple". Field
+	// "c".
+	Canonicalization string
+	Length           int64     // Body length to verify, default -1 for whole body. Field "l".
+	Identity         *Identity // AUID (agent/user id). If nil and an identity is needed, should be treated as an Identity without localpart and Domain from d= field. Field "i".
+	QueryMethods     []string  // For public key, currently known value is "dns/txt" (should be compared case-insensitively). If empty, dns/txt must be assumed. Field "q".
+	SignTime         int64     // Unix epoch. -1 if unset. Field "t".
+	ExpireTime       int64     // Unix epoch. -1 if unset. Field "x".
+	CopiedHeaders    []string  // Copied header fields. Field "z".
+}
+
+// Identity is used for the optional i= field in a DKIM-Signature header. It uses
+// the syntax of an email address, but does not necessarily represent one.
+type Identity struct {
+	Localpart *smtp.Localpart // Optional.
+	Domain    dns.Domain
+}
+
+// String returns a value for use in the i= DKIM-Signature field.
+func (i Identity) String() string {
+	s := "@" + i.Domain.ASCII
+	// We need localpart as pointer to indicate it is missing because localparts can be
+	// "" which we store (decoded) as empty string and we need to differentiate.
+	if i.Localpart != nil {
+		s = i.Localpart.String() + s
+	}
+	return s
+}
+
+func newSigWithDefaults() *Sig {
+	return &Sig{
+		Canonicalization: "simple/simple",
+		Length:           -1,
+		SignTime:         -1,
+		ExpireTime:       -1,
+	}
+}
+
+// Algorithm returns an algorithm string for use in the "a" field. E.g.
+// "ed25519-sha256".
+func (s Sig) Algorithm() string {
+	return s.AlgorithmSign + "-" + s.AlgorithmHash
+}
+
+// Header returns the DKIM-Signature header in string form, to be prepended to a
+// message, including DKIM-Signature field name and trailing \r\n.
+func (s *Sig) Header() (string, error) {
+	// ../rfc/6376:1021
+	// todo: make a higher-level writer that accepts pairs, and only folds to next line when needed.
+	w := &message.HeaderWriter{}
+	w.Addf("", "DKIM-Signature: v=%d;", s.Version)
+	// Domain names must always be in ASCII. ../rfc/6376:1115 ../rfc/6376:1187 ../rfc/6376:1303
+	w.Addf(" ", "d=%s;", s.Domain.ASCII)
+	w.Addf(" ", "s=%s;", s.Selector.ASCII)
+	if s.Identity != nil {
+		w.Addf(" ", "i=%s;", s.Identity.String()) // todo: Is utf-8 ok here?
+	}
+	w.Addf(" ", "a=%s;", s.Algorithm())
+
+	if s.Canonicalization != "" && !strings.EqualFold(s.Canonicalization, "simple") && !strings.EqualFold(s.Canonicalization, "simple/simple") {
+		w.Addf(" ", "c=%s;", s.Canonicalization)
+	}
+	if s.Length >= 0 {
+		w.Addf(" ", "l=%d;", s.Length)
+	}
+	if len(s.QueryMethods) > 0 && !(len(s.QueryMethods) == 1 && strings.EqualFold(s.QueryMethods[0], "dns/txt")) {
+		w.Addf(" ", "q=%s;", strings.Join(s.QueryMethods, ":"))
+	}
+	if s.SignTime >= 0 {
+		w.Addf(" ", "t=%d;", s.SignTime)
+	}
+	if s.ExpireTime >= 0 {
+		w.Addf(" ", "x=%d;", s.ExpireTime)
+	}
+
+	if len(s.SignedHeaders) > 0 {
+		for i, v := range s.SignedHeaders {
+			sep := ""
+			if i == 0 {
+				v = "h=" + v
+				sep = " "
+			}
+			if i < len(s.SignedHeaders)-1 {
+				v += ":"
+			} else if i == len(s.SignedHeaders)-1 {
+				v += ";"
+			}
+			w.Addf(sep, v)
+		}
+	}
+	if len(s.CopiedHeaders) > 0 {
+		// todo: wrap long headers? we can at least add FWS before the :
+		for i, v := range s.CopiedHeaders {
+			t := strings.SplitN(v, ":", 2)
+			if len(t) == 2 {
+				v = t[0] + ":" + packQpHdrValue(t[1])
+			} else {
+				return "", fmt.Errorf("invalid header in copied headers (z=): %q", v)
+			}
+			sep := ""
+			if i == 0 {
+				v = "z=" + v
+				sep = " "
+			}
+			if i < len(s.CopiedHeaders)-1 {
+				v += "|"
+			} else if i == len(s.CopiedHeaders)-1 {
+				v += ";"
+			}
+			w.Addf(sep, v)
+		}
+	}
+
+	w.Addf(" ", "bh=%s;", base64.StdEncoding.EncodeToString(s.BodyHash))
+
+	w.Addf(" ", "b=")
+	if len(s.Signature) > 0 {
+		w.AddWrap([]byte(base64.StdEncoding.EncodeToString(s.Signature)))
+	}
+	w.Add("\r\n")
+	return w.String(), nil
+}
+
+// Like quoted printable, but with "|" encoded as well.
+// We also encode ":" because it is used as separator in DKIM headers which can
+// cause trouble for "q", even though it is listed in dkim-safe-char,
+// ../rfc/6376:497.
+func packQpHdrValue(s string) string {
+	// ../rfc/6376:474
+	const hex = "0123456789ABCDEF"
+	var r string
+	for _, b := range []byte(s) {
+		if b > ' ' && b < 0x7f && b != ';' && b != '=' && b != '|' && b != ':' {
+			r += string(b)
+		} else {
+			r += "=" + string(hex[b>>4]) + string(hex[(b>>0)&0xf])
+		}
+	}
+	return r
+}
+
+var (
+	errSigHeader         = errors.New("not DKIM-Signature header")
+	errSigDuplicateTag   = errors.New("duplicate tag")
+	errSigMissingCRLF    = errors.New("missing crlf at end")
+	errSigExpired        = errors.New("signature timestamp (t=) must be before signature expiration (x=)")
+	errSigIdentityDomain = errors.New("identity domain (i=) not under domain (d=)")
+	errSigMissingTag     = errors.New("missing required tag")
+	errSigUnknownVersion = errors.New("unknown version")
+	errSigBodyHash       = errors.New("bad body hash size given algorithm")
+)
+
+// parseSignatures returns the parsed form of a DKIM-Signature header.
+//
+// buf must end in crlf, as it should have occurred in the mail message.
+//
+// The dkim signature with signature left empty ("b=") and without trailing
+// crlf is returned, for use in verification.
+func parseSignature(buf []byte, smtputf8 bool) (sig *Sig, verifySig []byte, err error) {
+	defer func() {
+		if x := recover(); x == nil {
+			return
+		} else if xerr, ok := x.(error); ok {
+			sig = nil
+			verifySig = nil
+			err = xerr
+		} else {
+			panic(x)
+		}
+	}()
+
+	xerrorf := func(format string, args ...any) {
+		panic(fmt.Errorf(format, args...))
+	}
+
+	if !bytes.HasSuffix(buf, []byte("\r\n")) {
+		xerrorf("%w", errSigMissingCRLF)
+	}
+	buf = buf[:len(buf)-2]
+
+	ds := newSigWithDefaults()
+	seen := map[string]struct{}{}
+	p := parser{s: string(buf), smtputf8: smtputf8}
+	name := p.xhdrName()
+	if !strings.EqualFold(name, "DKIM-Signature") {
+		xerrorf("%w", errSigHeader)
+	}
+	p.wsp()
+	p.xtake(":")
+	p.wsp()
+	// ../rfc/6376:655
+	// ../rfc/6376:656 ../rfc/6376-eid5070
+	// ../rfc/6376:658 ../rfc/6376-eid5070
+	for {
+		p.fws()
+		k := p.xtagName()
+		p.fws()
+		p.xtake("=")
+		// Special case for "b", see below.
+		if k != "b" {
+			p.fws()
+		}
+		// Keys are case-sensitive: ../rfc/6376:679
+		if _, ok := seen[k]; ok {
+			// Duplicates not allowed: ../rfc/6376:683
+			xerrorf("%w: %q", errSigDuplicateTag, k)
+			break
+		}
+		seen[k] = struct{}{}
+
+		// ../rfc/6376:1021
+		switch k {
+		case "v":
+			// ../rfc/6376:1025
+			ds.Version = int(p.xnumber(10))
+			if ds.Version != 1 {
+				xerrorf("%w: version %d", errSigUnknownVersion, ds.Version)
+			}
+		case "a":
+			// ../rfc/6376:1038
+			ds.AlgorithmSign, ds.AlgorithmHash = p.xalgorithm()
+		case "b":
+			// ../rfc/6376:1054
+			// To calculate the hash, we have to feed the DKIM-Signature header to the hash
+			// function, but with the value for "b=" (the signature) left out. The parser
+			// tracks all data that is read, except when drop is true.
+			// ../rfc/6376:997
+			// Surrounding whitespace must be cleared as well. ../rfc/6376:1659
+			// Note: The RFC says "surrounding" whitespace, but whitespace is only allowed
+			// before the value as part of the ABNF production for "b". Presumably the
+			// intention is to ignore the trailing "[FWS]" for the tag-spec production,
+			// ../rfc/6376:656
+			// Another indication is the term "value portion", ../rfc/6376:1667. It appears to
+			// mean everything after the "b=" part, instead of the actual value (either encoded
+			// or decoded).
+			p.drop = true
+			p.fws()
+			ds.Signature = p.xbase64()
+			p.fws()
+			p.drop = false
+		case "bh":
+			// ../rfc/6376:1076
+			ds.BodyHash = p.xbase64()
+		case "c":
+			// ../rfc/6376:1088
+			ds.Canonicalization = p.xcanonical()
+			// ../rfc/6376:810
+		case "d":
+			// ../rfc/6376:1105
+			ds.Domain = p.xdomain()
+		case "h":
+			// ../rfc/6376:1134
+			ds.SignedHeaders = p.xsignedHeaderFields()
+		case "i":
+			// ../rfc/6376:1171
+			id := p.xauid()
+			ds.Identity = &id
+		case "l":
+			// ../rfc/6376:1244
+			ds.Length = p.xbodyLength()
+		case "q":
+			// ../rfc/6376:1268
+			ds.QueryMethods = p.xqueryMethods()
+		case "s":
+			// ../rfc/6376:1300
+			ds.Selector = p.xselector()
+		case "t":
+			// ../rfc/6376:1310
+			ds.SignTime = p.xtimestamp()
+		case "x":
+			// ../rfc/6376:1327
+			ds.ExpireTime = p.xtimestamp()
+		case "z":
+			// ../rfc/6376:1361
+			ds.CopiedHeaders = p.xcopiedHeaderFields()
+		default:
+			// We must ignore unknown fields. ../rfc/6376:692 ../rfc/6376:1022
+			p.xchar() // ../rfc/6376-eid5070
+			for !p.empty() && !p.hasPrefix(";") {
+				p.xchar()
+			}
+		}
+		p.fws()
+
+		if p.empty() {
+			break
+		}
+		p.xtake(";")
+		if p.empty() {
+			break
+		}
+	}
+
+	// ../rfc/6376:2532
+	required := []string{"v", "a", "b", "bh", "d", "h", "s"}
+	for _, req := range required {
+		if _, ok := seen[req]; !ok {
+			xerrorf("%w: %q", errSigMissingTag, req)
+		}
+	}
+
+	if strings.EqualFold(ds.AlgorithmHash, "sha1") && len(ds.BodyHash) != 20 {
+		xerrorf("%w: got %d bytes, must be 20 for sha1", errSigBodyHash, len(ds.BodyHash))
+	} else if strings.EqualFold(ds.AlgorithmHash, "sha256") && len(ds.BodyHash) != 32 {
+		xerrorf("%w: got %d bytes, must be 32 for sha256", errSigBodyHash, len(ds.BodyHash))
+	}
+
+	// ../rfc/6376:1337
+	if ds.SignTime >= 0 && ds.ExpireTime >= 0 && ds.SignTime >= ds.ExpireTime {
+		xerrorf("%w", errSigExpired)
+	}
+
+	// Default identity is "@" plus domain. We don't set this value because we want to
+	// keep the distinction between absent value.
+	// ../rfc/6376:1172 ../rfc/6376:2537 ../rfc/6376:2541
+	if ds.Identity != nil && ds.Identity.Domain.ASCII != ds.Domain.ASCII && !strings.HasSuffix(ds.Identity.Domain.ASCII, "."+ds.Domain.ASCII) {
+		xerrorf("%w: identity domain %q not under domain %q", errSigIdentityDomain, ds.Identity.Domain.ASCII, ds.Domain.ASCII)
+	}
+
+	return ds, []byte(p.tracked), nil
+}

dkim/sig_test.go

@@ -0,0 +1,180 @@
+package dkim
+
+import (
+	"encoding/base64"
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/smtp"
+)
+
+func TestSig(t *testing.T) {
+	test := func(s string, smtputf8 bool, expSig *Sig, expErr error) {
+		t.Helper()
+
+		isParseErr := func(err error) bool {
+			_, ok := err.(parseErr)
+			return ok
+		}
+
+		sig, _, err := parseSignature([]byte(s), smtputf8)
+		if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) && !(isParseErr(err) && isParseErr(expErr)) {
+			t.Fatalf("got err %v, expected %v", err, expErr)
+		}
+		if !reflect.DeepEqual(sig, expSig) {
+			t.Fatalf("got sig %#v, expected %#v", sig, expSig)
+		}
+
+		if sig == nil {
+			return
+		}
+		h, err := sig.Header()
+		if err != nil {
+			t.Fatalf("making signature header: %v", err)
+		}
+		nsig, _, err := parseSignature([]byte(h), smtputf8)
+		if err != nil {
+			t.Fatalf("parse signature again: %v", err)
+		}
+		if !reflect.DeepEqual(nsig, sig) {
+			t.Fatalf("parsed signature again, got %#v, expected %#v", nsig, sig)
+		}
+	}
+
+	xbase64 := func(s string) []byte {
+		t.Helper()
+		buf, err := base64.StdEncoding.DecodeString(s)
+		if err != nil {
+			t.Fatalf("parsing base64: %v", err)
+		}
+		return buf
+	}
+
+	xdomain := func(s string) dns.Domain {
+		t.Helper()
+		d, err := dns.ParseDomain(s)
+		if err != nil {
+			t.Fatalf("parsing domain: %v", err)
+		}
+		return d
+	}
+
+	var empty smtp.Localpart
+	sig1 := &Sig{
+		Version:          1,
+		AlgorithmSign:    "ed25519",
+		AlgorithmHash:    "sha256",
+		Signature:        xbase64("dGVzdAo="),
+		BodyHash:         xbase64("LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q="),
+		Domain:           xdomain("mox.example"),
+		SignedHeaders:    []string{"from", "to", "cc", "bcc", "date", "subject", "message-id"},
+		Selector:         xdomain("test"),
+		Canonicalization: "simple/relaxed",
+		Length:           10,
+		Identity:         &Identity{&empty, xdomain("sub.mox.example")},
+		QueryMethods:     []string{"dns/txt", "other"},
+		SignTime:         10,
+		ExpireTime:       100,
+		CopiedHeaders:    []string{"From:<mjl@mox.example>", "Subject:test | with pipe"},
+	}
+	test("dkim-signature: v = 1 ; a=ed25519-sha256; s=test; d=mox.example; h=from:to:cc:bcc:date:subject:message-id; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q= ; c=simple/relaxed; l=10; i=\"\"@sub.mox.example; q= dns/txt:other; t=10; x=100; z=From:<mjl@mox.example>|Subject:test=20=7C=20with=20pipe; unknown = must be ignored \r\n", true, sig1, nil)
+
+	ulp := smtp.Localpart("møx")
+	sig2 := &Sig{
+		Version:          1,
+		AlgorithmSign:    "ed25519",
+		AlgorithmHash:    "sha256",
+		Signature:        xbase64("dGVzdAo="),
+		BodyHash:         xbase64("LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q="),
+		Domain:           xdomain("xn--mx-lka.example"), // møx.example
+		SignedHeaders:    []string{"from"},
+		Selector:         xdomain("xn--tst-bma"),                                     // tést
+		Identity:         &Identity{&ulp, xdomain("xn--tst-bma.xn--mx-lka.example")}, // tést.møx.example
+		Canonicalization: "simple/simple",
+		Length:           -1,
+		SignTime:         -1,
+		ExpireTime:       -1,
+	}
+	test("dkim-signature: v = 1 ; a=ed25519-sha256; s=xn--tst-bma; d=xn--mx-lka.example; h=from; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q= ; i=møx@xn--tst-bma.xn--mx-lka.example;\r\n", true, sig2, nil)
+	test("dkim-signature: v = 1 ; a=ed25519-sha256; s=xn--tst-bma; d=xn--mx-lka.example; h=from; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q= ; i=møx@xn--tst-bma.xn--mx-lka.example;\r\n", false, nil, parseErr("")) // No UTF-8 allowed.
+
+	multiatom := smtp.Localpart("a.b.c")
+	sig3 := &Sig{
+		Version:          1,
+		AlgorithmSign:    "ed25519",
+		AlgorithmHash:    "sha256",
+		Signature:        xbase64("dGVzdAo="),
+		BodyHash:         xbase64("LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q="),
+		Domain:           xdomain("mox.example"),
+		SignedHeaders:    []string{"from"},
+		Selector:         xdomain("test"),
+		Identity:         &Identity{&multiatom, xdomain("mox.example")},
+		Canonicalization: "simple/simple",
+		Length:           -1,
+		SignTime:         -1,
+		ExpireTime:       -1,
+	}
+	test("dkim-signature: v = 1 ; a=ed25519-sha256; s=test; d=mox.example; h=from; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q= ; i=a.b.c@mox.example\r\n", true, sig3, nil)
+
+	quotedlp := smtp.Localpart(`test "\test`)
+	sig4 := &Sig{
+		Version:          1,
+		AlgorithmSign:    "ed25519",
+		AlgorithmHash:    "sha256",
+		Signature:        xbase64("dGVzdAo="),
+		BodyHash:         xbase64("LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q="),
+		Domain:           xdomain("mox.example"),
+		SignedHeaders:    []string{"from"},
+		Selector:         xdomain("test"),
+		Identity:         &Identity{&quotedlp, xdomain("mox.example")},
+		Canonicalization: "simple/simple",
+		Length:           -1,
+		SignTime:         -1,
+		ExpireTime:       -1,
+	}
+	test("dkim-signature: v = 1 ; a=ed25519-sha256; s=test; d=mox.example; h=from; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q= ; i=\"test \\\"\\\\test\"@mox.example\r\n", true, sig4, nil)
+
+	test("", true, nil, errSigMissingCRLF)
+	test("other: ...\r\n", true, nil, errSigHeader)
+	test("dkim-signature: v=2\r\n", true, nil, errSigUnknownVersion)
+	test("dkim-signature: v=1\r\n", true, nil, errSigMissingTag)
+	test("dkim-signature: v=1;v=1\r\n", true, nil, errSigDuplicateTag)
+	test("dkim-signature: v=1; d=mox.example; i=@unrelated.example; s=test; a=ed25519-sha256; h=from; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q=\r\n", true, nil, errSigIdentityDomain)
+	test("dkim-signature: v=1; t=10; x=9; d=mox.example; s=test; a=ed25519-sha256; h=from; b=dGVzdAo=; bh=LjkN2rUhrS3zKXfH2vNgUzz5ERRJkgP9CURXBX0JP0Q=\r\n", true, nil, errSigExpired)
+	test("dkim-signature: v=1; d=møx.example\r\n", true, nil, parseErr("")) // Unicode domain not allowed.
+	test("dkim-signature: v=1; s=tést\r\n", true, nil, parseErr(""))        // Unicode selector not allowed.
+	test("dkim-signature: v=1; ;\r\n", true, nil, parseErr(""))             // Empty tag not allowed.
+	test("dkim-signature: v=1; \r\n", true, nil, parseErr(""))              // Cannot have whitespace after last colon.
+	test("dkim-signature: v=1; d=mox.example; s=test; a=ed25519-sha256; h=from; b=dGVzdAo=; bh=dGVzdAo=\r\n", true, nil, errSigBodyHash)
+	test("dkim-signature: v=1; d=mox.example; s=test; a=rsa-sha1; h=from; b=dGVzdAo=; bh=dGVzdAo=\r\n", true, nil, errSigBodyHash)
+}
+
+func TestCopiedHeadersSig(t *testing.T) {
+	// ../rfc/6376:1391
+	sigHeader := strings.ReplaceAll(`DKIM-Signature: v=1; a=rsa-sha256; d=example.net; s=brisbane;
+	c=simple; q=dns/txt; i=@eng.example.net;
+	t=1117574938; x=1118006938;
+	h=from:to:subject:date;
+	z=From:foo@eng.example.net|To:joe@example.com|
+	 Subject:demo=20run|Date:July=205,=202005=203:44:08=20PM=20-0700;
+	bh=MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=;
+	b=dzdVyOfAKCdLXdJOc9G2q8LoXSlEniSbav+yuU4zGeeruD00lszZVoG4ZHRNiYzR
+`, "\n", "\r\n")
+
+	sig, _, err := parseSignature([]byte(sigHeader), false)
+	if err != nil {
+		t.Fatalf("parsing dkim signature with copied headers: %v", err)
+	}
+	exp := []string{
+		"From:foo@eng.example.net",
+		"To:joe@example.com",
+		"Subject:demo run",
+		"Date:July 5, 2005 3:44:08 PM -0700",
+	}
+	if !reflect.DeepEqual(sig.CopiedHeaders, exp) {
+		t.Fatalf("copied headers, got %v, expected %v", sig.CopiedHeaders, exp)
+	}
+}

dkim/txt.go

@@ -0,0 +1,278 @@
+package dkim
+
+import (
+	"crypto/ed25519"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// Record is a DKIM DNS record, served on <selector>._domainkey.<domain> for a
+// given selector and domain (s= and d= in the DKIM-Signature).
+//
+// The record is a semicolon-separated list of "="-separated field value pairs.
+// Strings should be compared case-insensitively, e.g. k=ed25519 is equivalent to k=ED25519.
+//
+// Example:
+//
+//	v=DKIM1;h=sha256;k=ed25519;p=ln5zd/JEX4Jy60WAhUOv33IYm2YZMyTQAdr9stML504=
+type Record struct {
+	Version  string   // Version, fixed "DKIM1" (case sensitive). Field "v".
+	Hashes   []string // Acceptable hash algorithms, e.g. "sha1", "sha256". Optional, defaults to all algorithms. Field "h".
+	Key      string   // Key type, "rsa" or "ed25519". Optional, default "rsa". Field "k".
+	Notes    string   // Debug notes. Field "n".
+	Pubkey   []byte   // Public key, as base64 in record. If empty, the key has been revoked. Field "p".
+	Services []string // Service types. Optional, default "*" for all services. Other values: "email". Field "s".
+	Flags    []string // Flags, colon-separated. Optional, default is no flags. Other values: "y" for testing DKIM, "s" for "i=" must have same domain as "d" in signatures. Field "t".
+
+	PublicKey any `json:"-"` // Parsed form of public key, an *rsa.PublicKey or ed25519.PublicKey.
+}
+
+// ../rfc/6376:1438
+
+// ServiceAllowed returns whether service s is allowed by this key.
+//
+// The optional field "s" can specify purposes for which the key can be used. If
+// value was specified, both "*" and "email" are enough for use with DKIM.
+func (r *Record) ServiceAllowed(s string) bool {
+	if len(r.Services) == 0 {
+		return true
+	}
+	for _, ss := range r.Services {
+		if ss == "*" || strings.EqualFold(s, ss) {
+			return true
+		}
+	}
+	return false
+}
+
+// Record returns a DNS TXT record that should be served at
+// <selector>._domainkey.<domain>.
+//
+// Only values that are not the default values are included.
+func (r *Record) Record() (string, error) {
+	var l []string
+	add := func(s string) {
+		l = append(l, s)
+	}
+
+	if r.Version != "DKIM1" {
+		return "", fmt.Errorf("bad version, must be \"DKIM1\"")
+	}
+	add("v=DKIM1")
+	if len(r.Hashes) > 0 {
+		add("h=" + strings.Join(r.Hashes, ":"))
+	}
+	if r.Key != "" && !strings.EqualFold(r.Key, "rsa") {
+		add("k=" + r.Key)
+	}
+	if r.Notes != "" {
+		add("n=" + qpSection(r.Notes))
+	}
+	if len(r.Services) > 0 && (len(r.Services) != 1 || r.Services[0] != "*") {
+		add("s=" + strings.Join(r.Services, ":"))
+	}
+	if len(r.Flags) > 0 {
+		add("t=" + strings.Join(r.Flags, ":"))
+	}
+	// A missing public key is valid, it means the key has been revoked. ../rfc/6376:1501
+	pk := r.Pubkey
+	if len(pk) == 0 && r.PublicKey != nil {
+		switch k := r.PublicKey.(type) {
+		case *rsa.PublicKey:
+			var err error
+			pk, err = x509.MarshalPKIXPublicKey(k)
+			if err != nil {
+				return "", fmt.Errorf("marshal rsa public key: %v", err)
+			}
+		case ed25519.PublicKey:
+			pk = []byte(k)
+		default:
+			return "", fmt.Errorf("unknown public key type %T", r.PublicKey)
+		}
+	}
+	add("p=" + base64.StdEncoding.EncodeToString(pk))
+	return strings.Join(l, ";"), nil
+}
+
+func qpSection(s string) string {
+	const hex = "0123456789ABCDEF"
+
+	// ../rfc/2045:1260
+	var r string
+	for i, b := range []byte(s) {
+		if i > 0 && (b == ' ' || b == '\t') || b > ' ' && b < 0x7f && b != '=' {
+			r += string(rune(b))
+		} else {
+			r += "=" + string(hex[b>>4]) + string(hex[(b>>0)&0xf])
+		}
+	}
+	return r
+}
+
+var (
+	errRecordDuplicateTag     = errors.New("duplicate tag")
+	errRecordMissingField     = errors.New("missing field")
+	errRecordBadPublicKey     = errors.New("bad public key")
+	errRecordUnknownAlgorithm = errors.New("unknown algorithm")
+	errRecordVersionFirst     = errors.New("first field must be version")
+)
+
+// ParseRecord parses a DKIM DNS TXT record.
+//
+// If the record is a dkim record, but an error occurred, isdkim will be true and
+// err will be the error. Such errors must be treated differently from parse errors
+// where the record does not appear to be DKIM, which can happen with misconfigured
+// DNS (e.g. wildcard records).
+func ParseRecord(s string) (record *Record, isdkim bool, err error) {
+	defer func() {
+		x := recover()
+		if x == nil {
+			return
+		}
+		if xerr, ok := x.(error); ok {
+			record = nil
+			err = xerr
+			return
+		}
+		panic(x)
+	}()
+
+	xerrorf := func(format string, args ...any) {
+		panic(fmt.Errorf(format, args...))
+	}
+
+	record = &Record{
+		Version:  "DKIM1",
+		Key:      "rsa",
+		Services: []string{"*"},
+	}
+
+	p := parser{s: s, drop: true}
+	seen := map[string]struct{}{}
+	// ../rfc/6376:655
+	// ../rfc/6376:656 ../rfc/6376-eid5070
+	// ../rfc/6376:658 ../rfc/6376-eid5070
+	// ../rfc/6376:1438
+	for {
+		p.fws()
+		k := p.xtagName()
+		p.fws()
+		p.xtake("=")
+		p.fws()
+		// Keys are case-sensitive: ../rfc/6376:679
+		if _, ok := seen[k]; ok {
+			// Duplicates not allowed: ../rfc/6376:683
+			xerrorf("%w: %q", errRecordDuplicateTag, k)
+			break
+		}
+		seen[k] = struct{}{}
+		// Version must be the first.
+		switch k {
+		case "v":
+			// ../rfc/6376:1443
+			v := p.xtake("DKIM1")
+			// Version being set is a signal this appears to be a valid record. We must not
+			// treat e.g. DKIM1.1 as valid, so we explicitly check there is no more data before
+			// we decide this record is DKIM.
+			p.fws()
+			if !p.empty() {
+				p.xtake(";")
+			}
+			record.Version = v
+			if len(seen) != 1 {
+				// If version is present, it must be the first.
+				xerrorf("%w", errRecordVersionFirst)
+			}
+			isdkim = true
+			if p.empty() {
+				break
+			}
+			continue
+
+		case "h":
+			// ../rfc/6376:1463
+			record.Hashes = []string{p.xhyphenatedWord()}
+			for p.peekfws(":") {
+				p.fws()
+				p.xtake(":")
+				p.fws()
+				record.Hashes = append(record.Hashes, p.xhyphenatedWord())
+			}
+		case "k":
+			// ../rfc/6376:1478
+			record.Key = p.xhyphenatedWord()
+		case "n":
+			// ../rfc/6376:1491
+			record.Notes = p.xqpSection()
+		case "p":
+			// ../rfc/6376:1501
+			record.Pubkey = p.xbase64()
+		case "s":
+			// ../rfc/6376:1533
+			record.Services = []string{p.xhyphenatedWord()}
+			for p.peekfws(":") {
+				p.fws()
+				p.xtake(":")
+				p.fws()
+				record.Services = append(record.Services, p.xhyphenatedWord())
+			}
+		case "t":
+			// ../rfc/6376:1554
+			record.Flags = []string{p.xhyphenatedWord()}
+			for p.peekfws(":") {
+				p.fws()
+				p.xtake(":")
+				p.fws()
+				record.Flags = append(record.Flags, p.xhyphenatedWord())
+			}
+		default:
+			// We must ignore unknown fields. ../rfc/6376:692 ../rfc/6376:1439
+			for !p.empty() && !p.hasPrefix(";") {
+				p.xchar()
+			}
+		}
+
+		isdkim = true
+		p.fws()
+		if p.empty() {
+			break
+		}
+		p.xtake(";")
+		if p.empty() {
+			break
+		}
+	}
+
+	if _, ok := seen["p"]; !ok {
+		xerrorf("%w: public key", errRecordMissingField)
+	}
+
+	switch strings.ToLower(record.Key) {
+	case "", "rsa":
+		if len(record.Pubkey) == 0 {
+			// Revoked key, nothing to do.
+		} else if pk, err := x509.ParsePKIXPublicKey(record.Pubkey); err != nil {
+			xerrorf("%w: %s", errRecordBadPublicKey, err)
+		} else if _, ok := pk.(*rsa.PublicKey); !ok {
+			xerrorf("%w: got %T, need an RSA key", errRecordBadPublicKey, record.PublicKey)
+		} else {
+			record.PublicKey = pk
+		}
+	case "ed25519":
+		if len(record.Pubkey) == 0 {
+			// Revoked key, nothing to do.
+		} else if len(record.Pubkey) != ed25519.PublicKeySize {
+			xerrorf("%w: got %d bytes, need %d", errRecordBadPublicKey, len(record.Pubkey), ed25519.PublicKeySize)
+		} else {
+			record.PublicKey = ed25519.PublicKey(record.Pubkey)
+		}
+	default:
+		xerrorf("%w: %q", errRecordUnknownAlgorithm, record.Key)
+	}
+
+	return record, true, nil
+}

dkim/txt_test.go

@@ -0,0 +1,133 @@
+package dkim
+
+import (
+	"crypto/x509"
+	"encoding/base64"
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseRecord(t *testing.T) {
+	test := func(txt string, expRec *Record, expIsDKIM bool, expErr error) {
+		t.Helper()
+
+		isParseErr := func(err error) bool {
+			_, ok := err.(parseErr)
+			return ok
+		}
+
+		r, isdkim, err := ParseRecord(txt)
+		if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) && !(isParseErr(err) && isParseErr(expErr)) {
+			t.Fatalf("parsing record: got error %v %#v, expected %#v, txt %q", err, err, expErr, txt)
+		}
+		if isdkim != expIsDKIM {
+			t.Fatalf("got isdkim %v, expected %v", isdkim, expIsDKIM)
+		}
+		if r != nil && expRec != nil {
+			expRec.PublicKey = r.PublicKey
+		}
+		if !reflect.DeepEqual(r, expRec) {
+			t.Fatalf("got record %#v, expected %#v, for txt %q", r, expRec, txt)
+		}
+		if r != nil {
+			pk := r.Pubkey
+			for i := 0; i < 2; i++ {
+				ntxt, err := r.Record()
+				if err != nil {
+					t.Fatalf("making record: %v", err)
+				}
+				nr, _, _ := ParseRecord(ntxt)
+				r.Pubkey = pk
+				if !reflect.DeepEqual(r, nr) {
+					t.Fatalf("after packing and parsing, got %#v, expected %#v", nr, r)
+				}
+
+				// Generate again, now based on parsed public key.
+				pk = r.Pubkey
+				r.Pubkey = nil
+			}
+		}
+	}
+
+	xbase64 := func(s string) []byte {
+		t.Helper()
+		buf, err := base64.StdEncoding.DecodeString(s)
+		if err != nil {
+			t.Fatalf("parsing base64: %v", err)
+		}
+		return buf
+	}
+
+	test("", nil, false, parseErr(""))
+	test("v=DKIM1", nil, true, errRecordMissingField) // Missing p=.
+	test("p=; v=DKIM1", nil, true, errRecordVersionFirst)
+	test("v=DKIM1; p=; ", nil, true, parseErr(""))                                                    // Whitespace after last ; is not allowed.
+	test("v=dkim1; p=; ", nil, false, parseErr(""))                                                   // dkim1-value is case-sensitive.
+	test("v=DKIM1; p=JDcbZ0Hpba5NKXI4UAW3G0IDhhFOxhJTDybZEwe1FeA=", nil, true, errRecordBadPublicKey) // Not an rsa key.
+	test("v=DKIM1; p=; p=", nil, true, errRecordDuplicateTag)                                         // Duplicate tag.
+	test("v=DKIM1; k=ed25519; p=HbawiMnQXTCopHTkR0jlKQ==", nil, true, errRecordBadPublicKey)          // Short key.
+	test("v=DKIM1; k=unknown; p=", nil, true, errRecordUnknownAlgorithm)
+
+	empty := &Record{
+		Version:  "DKIM1",
+		Key:      "rsa",
+		Services: []string{"*"},
+		Pubkey:   []uint8{},
+	}
+	test("V=DKIM2; p=;", empty, true, nil) // Tag names are case-sensitive.
+
+	record := &Record{
+		Version:  "DKIM1",
+		Hashes:   []string{"sha1", "SHA256", "unknown"},
+		Key:      "ed25519",
+		Notes:    "notes...",
+		Pubkey:   xbase64("JDcbZ0Hpba5NKXI4UAW3G0IDhhFOxhJTDybZEwe1FeA="),
+		Services: []string{"email", "tlsrpt"},
+		Flags:    []string{"y", "t"},
+	}
+	test("v = DKIM1 ;   h\t=\tsha1 \t:\t SHA256:unknown\t;k=ed25519; n = notes...; p = JDc bZ0Hpb a5NK\tXI4UAW3G0IDhhFOxhJTDybZEwe1FeA=  ;s = email : tlsrpt; t = y\t: t; unknown = bogus;", record, true, nil)
+
+	edpkix, err := x509.MarshalPKIXPublicKey(record.PublicKey)
+	if err != nil {
+		t.Fatalf("marshal ed25519 public key")
+	}
+	recordx := &Record{
+		Version: "DKIM1",
+		Key:     "rsa",
+		Pubkey:  edpkix,
+	}
+	txtx, err := recordx.Record()
+	if err != nil {
+		t.Fatalf("making record: %v", err)
+	}
+	test(txtx, nil, true, errRecordBadPublicKey)
+
+	record2 := &Record{
+		Version:  "DKIM1",
+		Key:      "rsa",
+		Services: []string{"*"},
+		Pubkey:   xbase64("MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy3Z9ffZe8gUTJrdGuKj6IwEembmKYpp0jMa8uhudErcI4gFVUaFiiRWxc4jP/XR9NAEv3XwHm+CVcHu+L/n6VWt6g59U7vHXQicMfKGmEp2VplsgojNy/Y5X9HdVYM0azsI47NcJCDW9UVfeOHdOSgFME4F8dNtUKC4KTB2d1pqj/yixz+V8Sv8xkEyPfSRHcNXIw0LvelqJ1MRfN3hO/3uQSVrPYYk4SyV0b6wfnkQs28fpiIpGQvzlGI5WkrdOQT5k4YHaEvZDLNdwiMeVZOEL7dDoFs2mQsovm+tH0StUAZTnr61NLVFfD5V6Ip1V9zVtspPHvYSuOWwyArFZ9QIDAQAB"),
+	}
+	test("v=DKIM1;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy3Z9ffZe8gUTJrdGuKj6IwEembmKYpp0jMa8uhudErcI4gFVUaFiiRWxc4jP/XR9NAEv3XwHm+CVcHu+L/n6VWt6g59U7vHXQicMfKGmEp2VplsgojNy/Y5X9HdVYM0azsI47NcJCDW9UVfeOHdOSgFME4F8dNtUKC4KTB2d1pqj/yixz+V8Sv8xkEyPfSRHcNXIw0LvelqJ1MRfN3hO/3uQSVrPYYk4SyV0b6wfnkQs28fpiIpGQvzlGI5WkrdOQT5k4YHaEvZDLNdwiMeVZOEL7dDoFs2mQsovm+tH0StUAZTnr61NLVFfD5V6Ip1V9zVtspPHvYSuOWwyArFZ9QIDAQAB", record2, true, nil)
+
+}
+
+func TestQPSection(t *testing.T) {
+	var tests = []struct {
+		input  string
+		expect string
+	}{
+		{"test", "test"},
+		{"hi=", "hi=3D"},
+		{"hi there", "hi there"},
+		{" hi", "=20hi"},
+		{"t\x7f", "t=7F"},
+	}
+	for _, v := range tests {
+		r := qpSection(v.input)
+		if r != v.expect {
+			t.Fatalf("qpSection: input %q, expected %q, got %q", v.input, v.expect, r)
+		}
+	}
+}

dmarc/dmarc.go

@@ -0,0 +1,239 @@
+// Package dmarc implements DMARC (Domain-based Message Authentication,
+// Reporting, and Conformance; RFC 7489) verification.
+//
+// DMARC is a mechanism for verifying ("authenticating") the address in the "From"
+// message header, since users will look at that header to identify the sender of a
+// message. DMARC compares the "From"-(sub)domain against the SPF and/or
+// DKIM-validated domains, based on the DMARC policy that a domain has published in
+// DNS as TXT record under "_dmarc.<domain>". A DMARC policy can also ask for
+// feedback about evaluations by other email servers, for monitoring/debugging
+// problems with email delivery.
+package dmarc
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	mathrand "math/rand"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/mox/dkim"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/publicsuffix"
+	"github.com/mjl-/mox/spf"
+)
+
+var xlog = mlog.New("dmarc")
+
+var (
+	metricDMARCVerify = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "mox_dmarc_verify_duration_seconds",
+			Help:    "DMARC verify, including lookup, duration and result.",
+			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
+		},
+		[]string{
+			"status",
+			"reject", // yes/no
+			"use",    // yes/no, if policy is used after random selection
+		},
+	)
+)
+
+// link errata:
+// ../rfc/7489-eid5440 ../rfc/7489:1585
+
+// Lookup errors.
+var (
+	ErrNoRecord        = errors.New("dmarc: no dmarc dns record")
+	ErrMultipleRecords = errors.New("dmarc: multiple dmarc dns records") // Must also be treated as if domain does not implement DMARC.
+	ErrDNS             = errors.New("dmarc: dns lookup")
+	ErrSyntax          = errors.New("dmarc: malformed dmarc dns record")
+)
+
+// Status is the result of DMARC policy evaluation, for use in an Authentication-Results header.
+type Status string
+
+// ../rfc/7489:2339
+
+const (
+	StatusNone      Status = "none"      // No DMARC TXT DNS record found.
+	StatusPass      Status = "pass"      // SPF and/or DKIM pass with identifier alignment.
+	StatusFail      Status = "fail"      // Either both SPF and DKIM failed or identifier did not align with a pass.
+	StatusTemperror Status = "temperror" // Typically a DNS lookup. A later attempt may results in a conclusion.
+	StatusPermerror Status = "permerror" // Typically a malformed DMARC DNS record.
+)
+
+// Result is a DMARC policy evaluation.
+type Result struct {
+	// Whether to reject the message based on policies. If false, the message should
+	// not necessarily be accepted, e.g. due to reputation or content-based analysis.
+	Reject bool
+	// Result of DMARC validation. A message can fail validation, but still
+	// not be rejected, e.g. if the policy is "none".
+	Status Status
+	// Domain with the DMARC DNS record. May be the organizational domain instead of
+	// the domain in the From-header.
+	Domain dns.Domain
+	// Parsed DMARC record.
+	Record *Record
+	// Details about possible error condition, e.g. when parsing the DMARC record failed.
+	Err error
+}
+
+// Lookup looks up the DMARC TXT record at "_dmarc.<domain>" for the domain in the
+// "From"-header of a message.
+//
+// If no DMARC record is found for the "From"-domain, another lookup is done at
+// the organizational domain of the domain (if different). The organizational
+// domain is determined using the public suffix list. E.g. for
+// "sub.example.com", the organizational domain is "example.com". The returned
+// domain is the domain with the DMARC record.
+func Lookup(ctx context.Context, resolver dns.Resolver, from dns.Domain) (status Status, domain dns.Domain, record *Record, txt string, rerr error) {
+	log := xlog.WithContext(ctx)
+	start := time.Now()
+	defer func() {
+		log.Debugx("dmarc lookup result", rerr, mlog.Field("fromdomain", from), mlog.Field("status", status), mlog.Field("domain", domain), mlog.Field("record", record), mlog.Field("duration", time.Since(start)))
+	}()
+
+	// ../rfc/7489:859 ../rfc/7489:1370
+	domain = from
+	status, record, txt, err := lookupRecord(ctx, resolver, domain)
+	if status != StatusNone {
+		return status, domain, record, txt, err
+	}
+	if record == nil {
+		// ../rfc/7489:761 ../rfc/7489:1377
+		domain = publicsuffix.Lookup(ctx, from)
+		if domain == from {
+			return StatusNone, domain, nil, txt, err
+		}
+
+		status, record, txt, err = lookupRecord(ctx, resolver, domain)
+	}
+	return status, domain, record, txt, err
+}
+
+func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (Status, *Record, string, error) {
+	name := "_dmarc." + domain.ASCII + "."
+	txts, err := dns.WithPackage(resolver, "dmarc").LookupTXT(ctx, name)
+	if err != nil && !dns.IsNotFound(err) {
+		return StatusTemperror, nil, "", fmt.Errorf("%w: %s", ErrDNS, err)
+	}
+	var record *Record
+	var text string
+	var rerr error = ErrNoRecord
+	for _, txt := range txts {
+		r, isdmarc, err := ParseRecord(txt)
+		if !isdmarc {
+			// ../rfc/7489:1374
+			continue
+		} else if err != nil {
+			return StatusPermerror, nil, text, fmt.Errorf("%w: %s", ErrSyntax, err)
+		}
+		if record != nil {
+			// ../ ../rfc/7489:1388
+			return StatusNone, nil, "", ErrMultipleRecords
+		}
+		text = txt
+		record = r
+		rerr = nil
+	}
+	return StatusNone, record, text, rerr
+}
+
+// Verify evaluates the DMARC policy for the domain in the From-header of a
+// message given the DKIM and SPF evaluation results.
+//
+// applyRandomPercentage determines whether the records "pct" is honored. This
+// field specifies the percentage of messages the DMARC policy is applied to. It
+// is used for slow rollout of DMARC policies and should be honored during normal
+// email processing
+//
+// Verify always returns the result of verifying the DMARC policy
+// against the message (for inclusion in Authentication-Result headers).
+//
+// useResult indicates if the result should be applied in a policy decision.
+func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, applyRandomPercentage bool) (useResult bool, result Result) {
+	log := xlog.WithContext(ctx)
+	start := time.Now()
+	defer func() {
+		use := "no"
+		if useResult {
+			use = "yes"
+		}
+		reject := "no"
+		if result.Reject {
+			reject = "yes"
+		}
+		metricDMARCVerify.WithLabelValues(string(result.Status), reject, use).Observe(float64(time.Since(start)) / float64(time.Second))
+		log.Debugx("dmarc verify result", result.Err, mlog.Field("fromdomain", from), mlog.Field("dkimresults", dkimResults), mlog.Field("spfresult", spfResult), mlog.Field("status", result.Status), mlog.Field("reject", result.Reject), mlog.Field("use", useResult), mlog.Field("duration", time.Since(start)))
+	}()
+
+	status, recordDomain, record, _, err := Lookup(ctx, resolver, from)
+	if record == nil {
+		return false, Result{false, status, recordDomain, record, err}
+	}
+	result.Domain = recordDomain
+	result.Record = record
+
+	// Record can request sampling of messages to apply policy.
+	// See ../rfc/7489:1432
+	useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand.Intn(100) < record.Percentage
+
+	// We reject treat "quarantine" and "reject" the same. Thus, we also don't
+	// "downgrade" from reject to quarantine if this message was sampled out.
+	// ../rfc/7489:1446 ../rfc/7489:1024
+	if recordDomain != from && record.SubdomainPolicy != PolicyEmpty {
+		result.Reject = record.SubdomainPolicy != PolicyNone
+	} else {
+		result.Reject = record.Policy != PolicyNone
+	}
+
+	// ../rfc/7489:1338
+	result.Status = StatusFail
+	if spfResult == spf.StatusTemperror {
+		result.Status = StatusTemperror
+		result.Reject = false
+	}
+
+	// Below we can do a bunch of publicsuffix lookups. Cache the results, mostly to
+	// reduce log polution.
+	pubsuffixes := map[dns.Domain]dns.Domain{}
+	pubsuffix := func(name dns.Domain) dns.Domain {
+		if r, ok := pubsuffixes[name]; ok {
+			return r
+		}
+		r := publicsuffix.Lookup(ctx, name)
+		pubsuffixes[name] = r
+		return r
+	}
+
+	// ../rfc/7489:1319
+	// ../rfc/7489:544
+	if spfResult == spf.StatusPass && spfIdentity != nil && (*spfIdentity == from || result.Record.ASPF == "r" && pubsuffix(from) == pubsuffix(*spfIdentity)) {
+		result.Reject = false
+		result.Status = StatusPass
+		return
+	}
+
+	for _, dkimResult := range dkimResults {
+		if dkimResult.Status == dkim.StatusTemperror {
+			result.Reject = false
+			result.Status = StatusTemperror
+			continue
+		}
+		// ../rfc/7489:511
+		if dkimResult.Status == dkim.StatusPass && dkimResult.Sig != nil && (dkimResult.Sig.Domain == from || result.Record.ADKIM == "r" && pubsuffix(from) == pubsuffix(dkimResult.Sig.Domain)) {
+			// ../rfc/7489:535
+			result.Reject = false
+			result.Status = StatusPass
+			return
+		}
+	}
+	return
+}

dmarc/dmarc_test.go

@@ -0,0 +1,275 @@
+package dmarc
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/mjl-/mox/dkim"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/spf"
+)
+
+func TestLookup(t *testing.T) {
+	resolver := dns.MockResolver{
+		TXT: map[string][]string{
+			"_dmarc.simple.example.":    {"v=DMARC1; p=none;"},
+			"_dmarc.one.example.":       {"v=DMARC1; p=none;", "other"},
+			"_dmarc.temperror.example.": {"v=DMARC1; p=none;"},
+			"_dmarc.multiple.example.":  {"v=DMARC1; p=none;", "v=DMARC1; p=none;"},
+			"_dmarc.malformed.example.": {"v=DMARC1; p=none; bogus;"},
+			"_dmarc.example.com.":       {"v=DMARC1; p=none;"},
+		},
+		Fail: map[dns.Mockreq]struct{}{
+			{Type: "txt", Name: "_dmarc.temperror.example."}: {},
+		},
+	}
+
+	test := func(d string, expStatus Status, expDomain string, expRecord *Record, expErr error) {
+		t.Helper()
+
+		status, dom, record, _, err := Lookup(context.Background(), resolver, dns.Domain{ASCII: d})
+		if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
+			t.Fatalf("got err %#v, expected %#v", err, expErr)
+		}
+		expd := dns.Domain{ASCII: expDomain}
+		if status != expStatus || dom != expd || !reflect.DeepEqual(record, expRecord) {
+			t.Fatalf("got status %v, dom %v, record %#v, expected %v %v %#v", status, dom, record, expStatus, expDomain, expRecord)
+		}
+	}
+
+	r := DefaultRecord
+	r.Policy = PolicyNone
+	test("simple.example", StatusNone, "simple.example", &r, nil)
+	test("one.example", StatusNone, "one.example", &r, nil)
+	test("absent.example", StatusNone, "absent.example", nil, ErrNoRecord)
+	test("multiple.example", StatusNone, "multiple.example", nil, ErrMultipleRecords)
+	test("malformed.example", StatusPermerror, "malformed.example", nil, ErrSyntax)
+	test("temperror.example", StatusTemperror, "temperror.example", nil, ErrDNS)
+	test("sub.example.com", StatusNone, "example.com", &r, nil) // Policy published at organizational domain, public suffix.
+}
+
+func TestVerify(t *testing.T) {
+	resolver := dns.MockResolver{
+		TXT: map[string][]string{
+			"_dmarc.reject.example.":    {"v=DMARC1; p=reject"},
+			"_dmarc.strict.example.":    {"v=DMARC1; p=reject; adkim=s; aspf=s"},
+			"_dmarc.test.example.":      {"v=DMARC1; p=reject; pct=0"},
+			"_dmarc.subnone.example.":   {"v=DMARC1; p=reject; sp=none"},
+			"_dmarc.none.example.":      {"v=DMARC1; p=none"},
+			"_dmarc.malformed.example.": {"v=DMARC1; p=none; bogus"},
+			"_dmarc.example.com.":       {"v=DMARC1; p=reject"},
+		},
+		Fail: map[dns.Mockreq]struct{}{
+			{Type: "txt", Name: "_dmarc.temperror.example."}: {},
+		},
+	}
+
+	equalResult := func(got, exp Result) bool {
+		if reflect.DeepEqual(got, exp) {
+			return true
+		}
+		if got.Err != nil && exp.Err != nil && (got.Err == exp.Err || errors.Is(got.Err, exp.Err)) {
+			got.Err = nil
+			exp.Err = nil
+			return reflect.DeepEqual(got, exp)
+		}
+		return false
+	}
+
+	test := func(fromDom string, dkimResults []dkim.Result, spfResult spf.Status, spfIdentity *dns.Domain, expUseResult bool, expResult Result) {
+		t.Helper()
+
+		from, err := dns.ParseDomain(fromDom)
+		if err != nil {
+			t.Fatalf("parsing domain: %v", err)
+		}
+		useResult, result := Verify(context.Background(), resolver, from, dkimResults, spfResult, spfIdentity, true)
+		if useResult != expUseResult || !equalResult(result, expResult) {
+			t.Fatalf("verify: got useResult %v, result %#v, expected %v %#v", useResult, result, expUseResult, expResult)
+		}
+	}
+
+	// Basic case, reject policy and no dkim or spf results.
+	reject := DefaultRecord
+	reject.Policy = PolicyReject
+	test("reject.example",
+		[]dkim.Result{},
+		spf.StatusNone,
+		nil,
+		true, Result{true, StatusFail, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// Accept with spf pass.
+	test("reject.example",
+		[]dkim.Result{},
+		spf.StatusPass,
+		&dns.Domain{ASCII: "sub.reject.example"},
+		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// Accept with dkim pass.
+	test("reject.example",
+		[]dkim.Result{
+			{
+				Status: dkim.StatusPass,
+				Sig: &dkim.Sig{ // Just the minimum fields needed.
+					Domain: dns.Domain{ASCII: "sub.reject.example"},
+				},
+				Record: &dkim.Record{},
+			},
+		},
+		spf.StatusFail,
+		&dns.Domain{ASCII: "reject.example"},
+		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// Reject due to spf and dkim "strict".
+	strict := DefaultRecord
+	strict.Policy = PolicyReject
+	strict.ADKIM = AlignStrict
+	strict.ASPF = AlignStrict
+	test("strict.example",
+		[]dkim.Result{
+			{
+				Status: dkim.StatusPass,
+				Sig: &dkim.Sig{ // Just the minimum fields needed.
+					Domain: dns.Domain{ASCII: "sub.strict.example"},
+				},
+				Record: &dkim.Record{},
+			},
+		},
+		spf.StatusPass,
+		&dns.Domain{ASCII: "sub.strict.example"},
+		true, Result{true, StatusFail, dns.Domain{ASCII: "strict.example"}, &strict, nil},
+	)
+
+	// No dmarc policy, nothing to say.
+	test("absent.example",
+		[]dkim.Result{},
+		spf.StatusNone,
+		nil,
+		false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, ErrNoRecord},
+	)
+
+	// No dmarc policy, spf pass does nothing.
+	test("absent.example",
+		[]dkim.Result{},
+		spf.StatusPass,
+		&dns.Domain{ASCII: "absent.example"},
+		false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, ErrNoRecord},
+	)
+
+	none := DefaultRecord
+	none.Policy = PolicyNone
+	// Policy none results in no reject.
+	test("none.example",
+		[]dkim.Result{},
+		spf.StatusPass,
+		&dns.Domain{ASCII: "none.example"},
+		true, Result{false, StatusPass, dns.Domain{ASCII: "none.example"}, &none, nil},
+	)
+
+	// No actual reject due to pct=0.
+	testr := DefaultRecord
+	testr.Policy = PolicyReject
+	testr.Percentage = 0
+	test("test.example",
+		[]dkim.Result{},
+		spf.StatusNone,
+		nil,
+		false, Result{true, StatusFail, dns.Domain{ASCII: "test.example"}, &testr, nil},
+	)
+
+	// No reject if subdomain has "none" policy.
+	sub := DefaultRecord
+	sub.Policy = PolicyReject
+	sub.SubdomainPolicy = PolicyNone
+	test("sub.subnone.example",
+		[]dkim.Result{},
+		spf.StatusFail,
+		&dns.Domain{ASCII: "sub.subnone.example"},
+		true, Result{false, StatusFail, dns.Domain{ASCII: "subnone.example"}, &sub, nil},
+	)
+
+	// No reject if spf temperror and no other pass.
+	test("reject.example",
+		[]dkim.Result{},
+		spf.StatusTemperror,
+		&dns.Domain{ASCII: "mail.reject.example"},
+		true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// No reject if dkim temperror and no other pass.
+	test("reject.example",
+		[]dkim.Result{
+			{
+				Status: dkim.StatusTemperror,
+				Sig: &dkim.Sig{ // Just the minimum fields needed.
+					Domain: dns.Domain{ASCII: "sub.reject.example"},
+				},
+				Record: &dkim.Record{},
+			},
+		},
+		spf.StatusNone,
+		nil,
+		true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// No reject if spf temperror but still dkim pass.
+	test("reject.example",
+		[]dkim.Result{
+			{
+				Status: dkim.StatusPass,
+				Sig: &dkim.Sig{ // Just the minimum fields needed.
+					Domain: dns.Domain{ASCII: "sub.reject.example"},
+				},
+				Record: &dkim.Record{},
+			},
+		},
+		spf.StatusTemperror,
+		&dns.Domain{ASCII: "mail.reject.example"},
+		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// No reject if dkim temperror but still spf pass.
+	test("reject.example",
+		[]dkim.Result{
+			{
+				Status: dkim.StatusTemperror,
+				Sig: &dkim.Sig{ // Just the minimum fields needed.
+					Domain: dns.Domain{ASCII: "sub.reject.example"},
+				},
+				Record: &dkim.Record{},
+			},
+		},
+		spf.StatusPass,
+		&dns.Domain{ASCII: "mail.reject.example"},
+		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, nil},
+	)
+
+	// Bad DMARC record results in permerror without reject.
+	test("malformed.example",
+		[]dkim.Result{},
+		spf.StatusNone,
+		nil,
+		false, Result{false, StatusPermerror, dns.Domain{ASCII: "malformed.example"}, nil, ErrSyntax},
+	)
+
+	// DKIM domain that is higher-level than organizational can not result in a pass. ../rfc/7489:525
+	test("example.com",
+		[]dkim.Result{
+			{
+				Status: dkim.StatusPass,
+				Sig: &dkim.Sig{ // Just the minimum fields needed.
+					Domain: dns.Domain{ASCII: "com"},
+				},
+				Record: &dkim.Record{},
+			},
+		},
+		spf.StatusNone,
+		nil,
+		true, Result{true, StatusFail, dns.Domain{ASCII: "example.com"}, &reject, nil},
+	)
+}

dmarc/fuzz_test.go

@@ -0,0 +1,17 @@
+package dmarc
+
+import (
+	"testing"
+)
+
+func FuzzParseRecord(f *testing.F) {
+	f.Add("")
+	f.Add("V = DMARC1; P = reject ;\tSP=none; unknown \t=\t ignored-future-value \t ; adkim=s; aspf=s; rua=mailto:dmarc-feedback@example.com  ,\t\tmailto:tld-test@thirdparty.example.net!10m; RUF=mailto:auth-reports@example.com  ,\t\tmailto:tld-test@thirdparty.example.net!0G; RI = 123; FO = 0:1:d:s ; RF= afrf : other; Pct = 0")
+	f.Add("v=DMARC1; rua=mailto:dmarc-feedback@example.com!99999999999999999999999999999999999999999999999")
+	f.Fuzz(func(t *testing.T, s string) {
+		r, _, err := ParseRecord(s)
+		if err == nil {
+			_ = r.String()
+		}
+	})
+}

dmarc/parse.go

@@ -0,0 +1,343 @@
+package dmarc
+
+import (
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+type parseErr string
+
+func (e parseErr) Error() string {
+	return string(e)
+}
+
+// ParseRecord parses a DMARC TXT record.
+//
+// Fields and values are are case-insensitive in DMARC are returned in lower case
+// for easy comparison.
+//
+// DefaultRecord provides default values for tags not present in s.
+func ParseRecord(s string) (record *Record, isdmarc bool, rerr error) {
+	defer func() {
+		x := recover()
+		if x == nil {
+			return
+		}
+		if err, ok := x.(parseErr); ok {
+			rerr = err
+			return
+		}
+		panic(x)
+	}()
+
+	r := DefaultRecord
+	p := newParser(s)
+
+	// v= is required and must be first. ../rfc/7489:1099
+	p.xtake("v")
+	p.wsp()
+	p.xtake("=")
+	p.wsp()
+	r.Version = p.xtakecase("DMARC1")
+	p.wsp()
+	p.xtake(";")
+	isdmarc = true
+	seen := map[string]bool{}
+	for {
+		p.wsp()
+		if p.empty() {
+			break
+		}
+		W := p.xword()
+		w := strings.ToLower(W)
+		if seen[w] {
+			// RFC does not say anything about duplicate tags. They can only confuse, so we
+			// don't allow them.
+			p.xerrorf("duplicate tag %q", W)
+		}
+		seen[w] = true
+		p.wsp()
+		p.xtake("=")
+		p.wsp()
+		switch w {
+		default:
+			// ../rfc/7489:924 implies that we should know how to parse unknown tags.
+			// The formal definition at ../rfc/7489:1127 does not allow for unknown tags.
+			// We just parse until the next semicolon or end.
+			for !p.empty() {
+				if p.peek(';') {
+					break
+				}
+				p.xtaken(1)
+			}
+		case "p":
+			if len(seen) != 1 {
+				// ../rfc/7489:1105
+				p.xerrorf("p= (policy) must be first tag")
+			}
+			r.Policy = DMARCPolicy(p.xtakelist("none", "quarantine", "reject"))
+		case "sp":
+			r.SubdomainPolicy = DMARCPolicy(p.xkeyword())
+			// note: we check if the value is valid before returning.
+		case "rua":
+			r.AggregateReportAddresses = append(r.AggregateReportAddresses, p.xuri())
+			p.wsp()
+			for p.take(",") {
+				p.wsp()
+				r.AggregateReportAddresses = append(r.AggregateReportAddresses, p.xuri())
+				p.wsp()
+			}
+		case "ruf":
+			r.FailureReportAddresses = append(r.FailureReportAddresses, p.xuri())
+			p.wsp()
+			for p.take(",") {
+				p.wsp()
+				r.FailureReportAddresses = append(r.FailureReportAddresses, p.xuri())
+				p.wsp()
+			}
+		case "adkim":
+			r.ADKIM = Align(p.xtakelist("r", "s"))
+		case "aspf":
+			r.ASPF = Align(p.xtakelist("r", "s"))
+		case "ri":
+			r.AggregateReportingInterval = p.xnumber()
+		case "fo":
+			r.FailureReportingOptions = []string{p.xtakelist("0", "1", "d", "s")}
+			p.wsp()
+			for p.take(":") {
+				p.wsp()
+				r.FailureReportingOptions = append(r.FailureReportingOptions, p.xtakelist("0", "1", "d", "s"))
+				p.wsp()
+			}
+		case "rf":
+			r.ReportingFormat = []string{p.xkeyword()}
+			p.wsp()
+			for p.take(":") {
+				p.wsp()
+				r.ReportingFormat = append(r.ReportingFormat, p.xkeyword())
+				p.wsp()
+			}
+		case "pct":
+			r.Percentage = p.xnumber()
+			if r.Percentage > 100 {
+				p.xerrorf("bad percentage %d", r.Percentage)
+			}
+		}
+		p.wsp()
+		if !p.take(";") && !p.empty() {
+			p.xerrorf("expected ;")
+		}
+	}
+
+	// ../rfc/7489:1106 says "p" is required, but ../rfc/7489:1407 implies we must be
+	// able to parse a record without a "p" or with invalid "sp" tag.
+	sp := r.SubdomainPolicy
+	if !seen["p"] || sp != PolicyEmpty && sp != PolicyNone && sp != PolicyQuarantine && sp != PolicyReject {
+		if len(r.AggregateReportAddresses) > 0 {
+			r.Policy = PolicyNone
+			r.SubdomainPolicy = PolicyEmpty
+		} else {
+			p.xerrorf("invalid (subdomain)policy and no valid aggregate reporting address")
+		}
+	}
+
+	return &r, true, nil
+}
+
+type parser struct {
+	s     string
+	lower string
+	o     int
+}
+
+// toLower lower cases bytes that are A-Z. strings.ToLower does too much. and
+// would replace invalid bytes with unicode replacement characters, which would
+// break our requirement that offsets into the original and upper case strings
+// point to the same character.
+func toLower(s string) string {
+	r := []byte(s)
+	for i, c := range r {
+		if c >= 'A' && c <= 'Z' {
+			r[i] = c + 0x20
+		}
+	}
+	return string(r)
+}
+
+func newParser(s string) *parser {
+	return &parser{
+		s:     s,
+		lower: toLower(s),
+	}
+}
+
+func (p *parser) xerrorf(format string, args ...any) {
+	msg := fmt.Sprintf(format, args...)
+	if p.o < len(p.s) {
+		msg += fmt.Sprintf(" (remain %q)", p.s[p.o:])
+	}
+	panic(parseErr(msg))
+}
+
+func (p *parser) empty() bool {
+	return p.o >= len(p.s)
+}
+
+func (p *parser) peek(b byte) bool {
+	return p.o < len(p.s) && p.s[p.o] == b
+}
+
+// case insensitive prefix
+func (p *parser) prefix(s string) bool {
+	return strings.HasPrefix(p.lower[p.o:], s)
+}
+
+func (p *parser) take(s string) bool {
+	if p.prefix(s) {
+		p.o += len(s)
+		return true
+	}
+	return false
+}
+
+func (p *parser) xtaken(n int) string {
+	r := p.lower[p.o : p.o+n]
+	p.o += n
+	return r
+}
+
+func (p *parser) xtake(s string) string {
+	if !p.prefix(s) {
+		p.xerrorf("expected %q", s)
+	}
+	return p.xtaken(len(s))
+}
+
+func (p *parser) xtakecase(s string) string {
+	if !strings.HasPrefix(p.s[p.o:], s) {
+		p.xerrorf("expected %q", s)
+	}
+	r := p.s[p.o : p.o+len(s)]
+	p.o += len(s)
+	return r
+}
+
+// *WSP
+func (p *parser) wsp() {
+	for !p.empty() && (p.s[p.o] == ' ' || p.s[p.o] == '\t') {
+		p.o++
+	}
+}
+
+// take one of the strings in l.
+func (p *parser) xtakelist(l ...string) string {
+	for _, s := range l {
+		if p.prefix(s) {
+			return p.xtaken(len(s))
+		}
+	}
+	p.xerrorf("expected on one %v", l)
+	panic("not reached")
+}
+
+func (p *parser) xtakefn1case(fn func(byte, int) bool) string {
+	for i, b := range []byte(p.lower[p.o:]) {
+		if !fn(b, i) {
+			if i == 0 {
+				p.xerrorf("expected at least one char")
+			}
+			return p.xtaken(i)
+		}
+	}
+	if p.empty() {
+		p.xerrorf("expected at least 1 char")
+	}
+	r := p.s[p.o:]
+	p.o += len(r)
+	return r
+}
+
+// used for the tag keys.
+func (p *parser) xword() string {
+	return p.xtakefn1case(func(c byte, i int) bool {
+		return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
+	})
+}
+
+func (p *parser) xdigits() string {
+	return p.xtakefn1case(func(b byte, i int) bool {
+		return isdigit(b)
+	})
+}
+
+// ../rfc/7489:883
+// Syntax: ../rfc/7489:1132
+func (p *parser) xuri() URI {
+	// Ideally, we would simply parse an URI here. But a URI can contain a semicolon so
+	// could consume the rest of the DMARC record. Instead, we'll assume no one uses
+	// semicolons in URIs in DMARC records and first collect
+	// space/comma/semicolon/end-separated characters, then parse.
+	// ../rfc/3986:684
+	v := p.xtakefn1case(func(b byte, i int) bool {
+		return b != ',' && b != ' ' && b != '\t' && b != ';'
+	})
+	t := strings.SplitN(v, "!", 2)
+	u, err := url.Parse(t[0])
+	if err != nil {
+		p.xerrorf("parsing uri %q: %s", t[0], err)
+	}
+	if u.Scheme == "" {
+		p.xerrorf("missing scheme in uri")
+	}
+	uri := URI{
+		Address: t[0],
+	}
+	if len(t) == 2 {
+		o := t[1]
+		if o != "" {
+			c := o[len(o)-1]
+			switch c {
+			case 'k', 'K', 'm', 'M', 'g', 'G', 't', 'T':
+				uri.Unit = strings.ToLower(o[len(o)-1:])
+				o = o[:len(o)-1]
+			}
+		}
+		uri.MaxSize, err = strconv.ParseUint(o, 10, 64)
+		if err != nil {
+			p.xerrorf("parsing max size for uri: %s", err)
+		}
+	}
+	return uri
+}
+
+func (p *parser) xnumber() int {
+	digits := p.xdigits()
+	v, err := strconv.Atoi(digits)
+	if err != nil {
+		p.xerrorf("parsing %q: %s", digits, err)
+	}
+	return v
+}
+
+func (p *parser) xkeyword() string {
+	// ../rfc/7489:1195, keyword is imported from smtp.
+	// ../rfc/5321:2287
+	n := len(p.s) - p.o
+	return p.xtakefn1case(func(b byte, i int) bool {
+		return isalphadigit(b) || (b == '-' && i < n-1 && isalphadigit(p.s[p.o+i+1]))
+	})
+}
+
+func isdigit(b byte) bool {
+	return b >= '0' && b <= '9'
+}
+
+func isalpha(b byte) bool {
+	return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z'
+}
+
+func isalphadigit(b byte) bool {
+	return isdigit(b) || isalpha(b)
+}

dmarc/parse_test.go

@@ -0,0 +1,142 @@
+package dmarc
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParse(t *testing.T) {
+	// ../rfc/7489:3224
+
+	// bad cases
+	bad := func(s string) {
+		t.Helper()
+		_, _, err := ParseRecord(s)
+		if err == nil {
+			t.Fatalf("got parse success, expected error")
+		}
+	}
+	bad("")
+	bad("v=")
+	bad("v=DMARC12")                                           // "2" leftover
+	bad("v=DMARC1")                                            // semicolon required
+	bad("v=dmarc1; p=none")                                    // dmarc1 is case-sensitive
+	bad("v=DMARC1 p=none")                                     // missing ;
+	bad("v=DMARC1;")                                           // missing p, no rua
+	bad("v=DMARC1; sp=invalid")                                // invalid sp, no rua
+	bad("v=DMARC1; sp=reject; p=reject")                       // p must be directly after v
+	bad("v=DMARC1; p=none; p=none")                            // dup
+	bad("v=DMARC1; p=none; p=reject")                          // dup
+	bad("v=DMARC1;;")                                          // missing tag
+	bad("v=DMARC1; adkim=x")                                   // bad value
+	bad("v=DMARC1; aspf=123")                                  // bad value
+	bad("v=DMARC1; ri=")                                       // missing value
+	bad("v=DMARC1; ri=-1")                                     // invalid, must be >= 0
+	bad("v=DMARC1; ri=99999999999999999999999999999999999999") // does not fit in int
+	bad("v=DMARC1; ri=123bad")                                 // leftover data
+	bad("v=DMARC1; ri=bad")                                    // not a number
+	bad("v=DMARC1; fo=")
+	bad("v=DMARC1; fo=01")
+	bad("v=DMARC1; fo=bad")
+	bad("v=DMARC1; rf=bad-trailing-dash-")
+	bad("v=DMARC1; rf=")
+	bad("v=DMARC1; rf=bad.non-alphadigitdash")
+	bad("v=DMARC1; p=badvalue")
+	bad("v=DMARC1; sp=bad")
+	bad("v=DMARC1; pct=110")
+	bad("v=DMARC1; pct=bogus")
+	bad("v=DMARC1; pct=")
+	bad("v=DMARC1; rua=")
+	bad("v=DMARC1; rua=bogus")
+	bad("v=DMARC1; rua=mailto:dmarc-feedback@example.com!")
+	bad("v=DMARC1; rua=mailto:dmarc-feedback@example.com!99999999999999999999999999999999999999999999999")
+	bad("v=DMARC1; rua=mailto:dmarc-feedback@example.com!10p")
+
+	valid := func(s string, exp Record) {
+		t.Helper()
+
+		r, _, err := ParseRecord(s)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if !reflect.DeepEqual(r, &exp) {
+			t.Fatalf("got:\n%#v\nexpected:\n%#v", r, &exp)
+		}
+	}
+
+	// Return a record with default values, and overrides from r. Only for the fields used below.
+	record := func(r Record) Record {
+		rr := DefaultRecord
+		if r.Policy != "" {
+			rr.Policy = r.Policy
+		}
+		if r.AggregateReportAddresses != nil {
+			rr.AggregateReportAddresses = r.AggregateReportAddresses
+		}
+		if r.FailureReportAddresses != nil {
+			rr.FailureReportAddresses = r.FailureReportAddresses
+		}
+		if r.Percentage != 0 {
+			rr.Percentage = r.Percentage
+		}
+		return rr
+	}
+
+	valid("v=DMARC1; rua=mailto:mjl@mox.example", record(Record{
+		Policy: "none",
+		AggregateReportAddresses: []URI{
+			{Address: "mailto:mjl@mox.example"},
+		},
+	})) // ../rfc/7489:1407
+	valid("v=DMARC1; p=reject; sp=invalid; rua=mailto:mjl@mox.example", record(Record{
+		Policy: "none",
+		AggregateReportAddresses: []URI{
+			{Address: "mailto:mjl@mox.example"},
+		},
+	})) // ../rfc/7489:1407
+	valid("v=DMARC1; p=none; rua=mailto:dmarc-feedback@example.com", record(Record{
+		Policy: "none",
+		AggregateReportAddresses: []URI{
+			{Address: "mailto:dmarc-feedback@example.com"},
+		},
+	}))
+	valid("v=DMARC1; p=none; rua=mailto:dmarc-feedback@example.com;ruf=mailto:auth-reports@example.com", record(Record{
+		Policy: "none",
+		AggregateReportAddresses: []URI{
+			{Address: "mailto:dmarc-feedback@example.com"},
+		},
+		FailureReportAddresses: []URI{
+			{Address: "mailto:auth-reports@example.com"},
+		},
+	}))
+	valid("v=DMARC1; p=quarantine; rua=mailto:dmarc-feedback@example.com,mailto:tld-test@thirdparty.example.net!10m; pct=25", record(Record{
+		Policy: "quarantine",
+		AggregateReportAddresses: []URI{
+			{Address: "mailto:dmarc-feedback@example.com"},
+			{Address: "mailto:tld-test@thirdparty.example.net", MaxSize: 10, Unit: "m"},
+		},
+		Percentage: 25,
+	}))
+
+	valid("V = DMARC1 ; P = reject ;\tSP=none; unknown \t=\t ignored-future-value \t ; adkim=s; aspf=s; rua=mailto:dmarc-feedback@example.com  ,\t\tmailto:tld-test@thirdparty.example.net!10m; RUF=mailto:auth-reports@example.com  ,\t\tmailto:tld-test@thirdparty.example.net!0G; RI = 123; FO = 0:1:d:s ; RF= afrf : other; Pct = 0",
+		Record{
+			Version:         "DMARC1",
+			Policy:          "reject",
+			SubdomainPolicy: "none",
+			ADKIM:           "s",
+			ASPF:            "s",
+			AggregateReportAddresses: []URI{
+				{Address: "mailto:dmarc-feedback@example.com"},
+				{Address: "mailto:tld-test@thirdparty.example.net", MaxSize: 10, Unit: "m"},
+			},
+			FailureReportAddresses: []URI{
+				{Address: "mailto:auth-reports@example.com"},
+				{Address: "mailto:tld-test@thirdparty.example.net", MaxSize: 0, Unit: "g"},
+			},
+			AggregateReportingInterval: 123,
+			FailureReportingOptions:    []string{"0", "1", "d", "s"},
+			ReportingFormat:            []string{"afrf", "other"},
+			Percentage:                 0,
+		},
+	)
+}

dmarc/txt.go

@@ -0,0 +1,127 @@
+package dmarc
+
+import (
+	"fmt"
+	"strings"
+)
+
+// todo: DMARCPolicy should be named just Policy, but this is causing conflicting types in sherpadoc output. should somehow get the dmarc-prefix only in the sherpadoc.
+
+// Policy as used in DMARC DNS record for "p=" or "sp=".
+type DMARCPolicy string
+
+// ../rfc/7489:1157
+
+const (
+	PolicyEmpty      DMARCPolicy = "" // Only for the optional Record.SubdomainPolicy.
+	PolicyNone       DMARCPolicy = "none"
+	PolicyQuarantine DMARCPolicy = "quarantine"
+	PolicyReject     DMARCPolicy = "reject"
+)
+
+// URI is a destination address for reporting.
+type URI struct {
+	Address string // Should start with "mailto:".
+	MaxSize uint64 // Optional maximum message size, subject to Unit.
+	Unit    string // "" (b), "k", "g", "t" (case insensitive), unit size, where k is 2^10 etc.
+}
+
+// String returns a string representation of the URI for inclusion in a DMARC
+// record.
+func (u URI) String() string {
+	s := u.Address
+	s = strings.ReplaceAll(s, ",", "%2C")
+	s = strings.ReplaceAll(s, "!", "%21")
+	if u.MaxSize > 0 {
+		s += fmt.Sprintf("%d", u.MaxSize)
+	}
+	s += u.Unit
+	return s
+}
+
+// ../rfc/7489:1127
+
+// Align specifies the required alignment of a domain name.
+type Align string
+
+const (
+	AlignStrict  Align = "s" // Strict requires an exact domain name match.
+	AlignRelaxed Align = "r" // Relaxed requires either an exact or subdomain name match.
+)
+
+// Record is a DNS policy or reporting record.
+//
+// Example:
+//
+//	v=DMARC1; p=reject; rua=mailto:postmaster@mox.example
+type Record struct {
+	Version                    string      // "v=DMARC1"
+	Policy                     DMARCPolicy // Required, for "p=".
+	SubdomainPolicy            DMARCPolicy // Like policy but for subdomains. Optional, for "sp=".
+	AggregateReportAddresses   []URI       // Optional, for "rua=".
+	FailureReportAddresses     []URI       // Optional, for "ruf="
+	ADKIM                      Align       // "r" (default) for relaxed or "s" for simple. For "adkim=".
+	ASPF                       Align       // "r" (default) for relaxed or "s" for simple. For "aspf=".
+	AggregateReportingInterval int         // Default 86400. For "ri="
+	FailureReportingOptions    []string    // "0" (default), "1", "d", "s". For "fo=".
+	ReportingFormat            []string    // "afrf" (default). Ffor "rf=".
+	Percentage                 int         // Between 0 and 100, default 100. For "pct=".
+}
+
+// DefaultRecord holds the defaults for a DMARC record.
+var DefaultRecord = Record{
+	Version:                    "DMARC1",
+	ADKIM:                      "r",
+	ASPF:                       "r",
+	AggregateReportingInterval: 86400,
+	FailureReportingOptions:    []string{"0"},
+	ReportingFormat:            []string{"afrf"},
+	Percentage:                 100,
+}
+
+// String returns the DMARC record for use as DNS TXT record.
+func (r Record) String() string {
+	b := &strings.Builder{}
+	b.WriteString("v=" + r.Version)
+
+	wrote := false
+	write := func(do bool, tag, value string) {
+		if do {
+			fmt.Fprintf(b, ";%s=%s", tag, value)
+			wrote = true
+		}
+	}
+	write(r.Policy != "", "p", string(r.Policy))
+	write(r.SubdomainPolicy != "", "sp", string(r.SubdomainPolicy))
+	if len(r.AggregateReportAddresses) > 0 {
+		l := make([]string, len(r.AggregateReportAddresses))
+		for i, a := range r.AggregateReportAddresses {
+			l[i] = a.String()
+		}
+		s := strings.Join(l, ",")
+		write(true, "rua", s)
+	}
+	if len(r.FailureReportAddresses) > 0 {
+		l := make([]string, len(r.FailureReportAddresses))
+		for i, a := range r.FailureReportAddresses {
+			l[i] = a.String()
+		}
+		s := strings.Join(l, ",")
+		write(true, "ruf", s)
+	}
+	write(r.ADKIM != "", "adkim", string(r.ADKIM))
+	write(r.ASPF != "", "aspf", string(r.ASPF))
+	write(r.AggregateReportingInterval != DefaultRecord.AggregateReportingInterval, "ri", fmt.Sprintf("%d", r.AggregateReportingInterval))
+	if len(r.FailureReportingOptions) > 1 || (len(r.FailureReportingOptions) == 1 && r.FailureReportingOptions[0] != "0") {
+		write(true, "fo", strings.Join(r.FailureReportingOptions, ":"))
+	}
+	if len(r.ReportingFormat) > 1 || (len(r.ReportingFormat) == 1 && strings.EqualFold(r.ReportingFormat[0], "afrf")) {
+		write(true, "rf", strings.Join(r.FailureReportingOptions, ":"))
+	}
+	write(r.Percentage != 100, "pct", fmt.Sprintf("%d", r.Percentage))
+
+	if !wrote {
+		b.WriteString(";")
+	}
+	return b.String()
+}

dmarcdb/db.go

@@ -0,0 +1,186 @@
+// Package dmarcdb stores incoming DMARC reports.
+//
+// With DMARC, a domain can request emails with DMARC verification results by
+// remote mail servers to be sent to a specified address. Mox parses such
+// reports, stores them in its database and makes them available through its
+// admin web interface.
+package dmarcdb
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/dmarcrpt"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+)
+
+var xlog = mlog.New("dmarcdb")
+
+var (
+	dmarcDB *bstore.DB
+	mutex   sync.Mutex
+)
+
+var (
+	metricEvaluated = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mox_dmarcdb_policy_evaluated_total",
+			Help: "Number of policy evaluations.",
+		},
+		// We only register validated domains for which we have a config.
+		[]string{"domain", "disposition", "dkim", "spf"},
+	)
+	metricDKIM = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mox_dmarcdb_dkim_result_total",
+			Help: "Number of DKIM results.",
+		},
+		[]string{"result"},
+	)
+	metricSPF = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mox_dmarcdb_spf_result_total",
+			Help: "Number of SPF results.",
+		},
+		[]string{"result"},
+	)
+)
+
+// DomainFeedback is a single report stored in the database.
+type DomainFeedback struct {
+	ID int64
+	// Domain where DMARC DNS record was found, could be organizational domain.
+	Domain string `bstore:"index"`
+	// Domain in From-header.
+	FromDomain string `bstore:"index"`
+	dmarcrpt.Feedback
+}
+
+func database() (rdb *bstore.DB, rerr error) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if dmarcDB == nil {
+		p := mox.DataDirPath("dmarcrpt.db")
+		os.MkdirAll(filepath.Dir(p), 0770)
+		db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{})
+		if err != nil {
+			return nil, err
+		}
+		dmarcDB = db
+	}
+	return dmarcDB, nil
+}
+
+// Init opens the database.
+func Init() error {
+	_, err := database()
+	return err
+}
+
+// AddReport adds a DMARC aggregate feedback report from an email to the database,
+// and updates prometheus metrics.
+//
+// fromDomain is the domain in the report message From header.
+func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error {
+	db, err := database()
+	if err != nil {
+		return err
+	}
+
+	d, err := dns.ParseDomain(f.PolicyPublished.Domain)
+	if err != nil {
+		return fmt.Errorf("parsing domain in report: %v", err)
+	}
+
+	df := DomainFeedback{0, d.Name(), fromDomain.Name(), *f}
+	if err := db.Insert(&df); err != nil {
+		return err
+	}
+
+	for _, r := range f.Records {
+		for _, dkim := range r.AuthResults.DKIM {
+			count := r.Row.Count
+			if count > 0 {
+				metricDKIM.With(prometheus.Labels{
+					"result": string(dkim.Result),
+				}).Add(float64(count))
+			}
+		}
+
+		for _, spf := range r.AuthResults.SPF {
+			count := r.Row.Count
+			if count > 0 {
+				metricSPF.With(prometheus.Labels{
+					"result": string(spf.Result),
+				}).Add(float64(count))
+			}
+		}
+
+		count := r.Row.Count
+		if count > 0 {
+			pe := r.Row.PolicyEvaluated
+			metricEvaluated.With(prometheus.Labels{
+				"domain":      f.PolicyPublished.Domain,
+				"disposition": string(pe.Disposition),
+				"dkim":        string(pe.DKIM),
+				"spf":         string(pe.SPF),
+			}).Add(float64(count))
+		}
+	}
+	return nil
+}
+
+// Records returns all reports in the database.
+func Records(ctx context.Context) ([]DomainFeedback, error) {
+	db, err := database()
+	if err != nil {
+		return nil, err
+	}
+
+	return bstore.QueryDB[DomainFeedback](db).List()
+}
+
+// RecordID returns the report for the ID.
+func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
+	db, err := database()
+	if err != nil {
+		return DomainFeedback{}, err
+	}
+
+	e := DomainFeedback{ID: id}
+	err = db.Get(&e)
+	return e, err
+}
+
+// RecordsPeriodDomain returns the reports overlapping start and end, for the given
+// domain. If domain is empty, all records match for domain.
+func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) {
+	db, err := database()
+	if err != nil {
+		return nil, err
+	}
+
+	s := start.Unix()
+	e := end.Unix()
+
+	q := bstore.QueryDB[DomainFeedback](db)
+	if domain != "" {
+		q.FilterNonzero(DomainFeedback{Domain: domain})
+	}
+	q.FilterFn(func(d DomainFeedback) bool {
+		m := d.Feedback.ReportMetadata.DateRange
+		return m.Begin >= s && m.Begin < e || m.End > s && m.End <= e
+	})
+	return q.List()
+}

dmarcdb/db_test.go

@@ -0,0 +1,108 @@
+package dmarcdb
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/dmarcrpt"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mox-"
+)
+
+func TestDMARCDB(t *testing.T) {
+	mox.ConfigStaticPath = "../testdata/dmarcdb/fake.conf"
+	mox.Conf.Static.DataDir = "."
+
+	dbpath := mox.DataDirPath("dmarcrpt.db")
+	os.MkdirAll(filepath.Dir(dbpath), 0770)
+	defer os.Remove(dbpath)
+
+	if err := Init(); err != nil {
+		t.Fatalf("init database: %s", err)
+	}
+
+	feedback := &dmarcrpt.Feedback{
+		ReportMetadata: dmarcrpt.ReportMetadata{
+			OrgName:          "google.com",
+			Email:            "noreply-dmarc-support@google.com",
+			ExtraContactInfo: "https://support.google.com/a/answer/2466580",
+			ReportID:         "10051505501689795560",
+			DateRange: dmarcrpt.DateRange{
+				Begin: 1596412800,
+				End:   1596499199,
+			},
+		},
+		PolicyPublished: dmarcrpt.PolicyPublished{
+			Domain:          "example.org",
+			ADKIM:           "r",
+			ASPF:            "r",
+			Policy:          "reject",
+			SubdomainPolicy: "reject",
+			Percentage:      100,
+		},
+		Records: []dmarcrpt.ReportRecord{
+			{
+				Row: dmarcrpt.Row{
+					SourceIP: "127.0.0.1",
+					Count:    1,
+					PolicyEvaluated: dmarcrpt.PolicyEvaluated{
+						Disposition: dmarcrpt.DispositionNone,
+						DKIM:        dmarcrpt.DMARCPass,
+						SPF:         dmarcrpt.DMARCPass,
+					},
+				},
+				Identifiers: dmarcrpt.Identifiers{
+					HeaderFrom: "example.org",
+				},
+				AuthResults: dmarcrpt.AuthResults{
+					DKIM: []dmarcrpt.DKIMAuthResult{
+						{
+							Domain:   "example.org",
+							Result:   dmarcrpt.DKIMPass,
+							Selector: "example",
+						},
+					},
+					SPF: []dmarcrpt.SPFAuthResult{
+						{
+							Domain: "example.org",
+							Result: dmarcrpt.SPFPass,
+						},
+					},
+				},
+			},
+		},
+	}
+	if err := AddReport(context.Background(), feedback, dns.Domain{ASCII: "google.com"}); err != nil {
+		t.Fatalf("adding report: %s", err)
+	}
+
+	records, err := Records(context.Background())
+	if err != nil || len(records) != 1 || !reflect.DeepEqual(&records[0].Feedback, feedback) {
+		t.Fatalf("records: got err %v, records %#v, expected no error, single record with feedback %#v", err, records, feedback)
+	}
+
+	record, err := RecordID(context.Background(), records[0].ID)
+	if err != nil || !reflect.DeepEqual(&record.Feedback, feedback) {
+		t.Fatalf("record id: got err %v, record %#v, expected feedback %#v", err, record, feedback)
+	}
+
+	start := time.Unix(1596412800, 0)
+	end := time.Unix(1596499199, 0)
+	records, err = RecordsPeriodDomain(context.Background(), start, end, "example.org")
+	if err != nil || len(records) != 1 || !reflect.DeepEqual(&records[0].Feedback, feedback) {
+		t.Fatalf("records: got err %v, records %#v, expected no error, single record with feedback %#v", err, records, feedback)
+	}
+
+	records, err = RecordsPeriodDomain(context.Background(), end, end, "example.org")
+	if err != nil || len(records) != 0 {
+		t.Fatalf("records: got err %v, records %#v, expected no error and no records", err, records)
+	}
+	records, err = RecordsPeriodDomain(context.Background(), start, end, "other.example")
+	if err != nil || len(records) != 0 {
+		t.Fatalf("records: got err %v, records %#v, expected no error and no records", err, records)
+	}
+}

dmarcrpt/feedback.go

@@ -0,0 +1,157 @@
+package dmarcrpt
+
+// Initially generated by xsdgen, then modified.
+
+// Feedback is the top-level XML field returned.
+type Feedback struct {
+	Version         string          `xml:"version"`
+	ReportMetadata  ReportMetadata  `xml:"report_metadata"`
+	PolicyPublished PolicyPublished `xml:"policy_published"`
+	Records         []ReportRecord  `xml:"record"`
+}
+
+type ReportMetadata struct {
+	OrgName          string    `xml:"org_name"`
+	Email            string    `xml:"email"`
+	ExtraContactInfo string    `xml:"extra_contact_info,omitempty"`
+	ReportID         string    `xml:"report_id"`
+	DateRange        DateRange `xml:"date_range"`
+	Errors           []string  `xml:"error,omitempty"`
+}
+
+type DateRange struct {
+	Begin int64 `xml:"begin"`
+	End   int64 `xml:"end"`
+}
+
+// PolicyPublished is the policy as found in DNS for the domain.
+type PolicyPublished struct {
+	Domain           string      `xml:"domain"`
+	ADKIM            Alignment   `xml:"adkim,omitempty"`
+	ASPF             Alignment   `xml:"aspf,omitempty"`
+	Policy           Disposition `xml:"p"`
+	SubdomainPolicy  Disposition `xml:"sp"`
+	Percentage       int         `xml:"pct"`
+	ReportingOptions string      `xml:"fo"`
+}
+
+// Alignment is the identifier alignment.
+type Alignment string
+
+const (
+	AlignmentRelaxed Alignment = "r" // Subdomains match the DMARC from-domain.
+	AlignmentStrict  Alignment = "s" // Only exact from-domain match.
+)
+
+// Disposition is the requested action for a DMARC fail as specified in the
+// DMARC policy in DNS.
+type Disposition string
+
+const (
+	DispositionNone       Disposition = "none"
+	DispositionQuarantine Disposition = "quarantine"
+	DispositionReject     Disposition = "reject"
+)
+
+type ReportRecord struct {
+	Row         Row         `xml:"row"`
+	Identifiers Identifiers `xml:"identifiers"`
+	AuthResults AuthResults `xml:"auth_results"`
+}
+
+type Row struct {
+	// SourceIP must match the pattern ((1?[0-9]?[0-9]|2[0-4][0-9]|25[0-5]).){3}
+	// (1?[0-9]?[0-9]|2[0-4][0-9]|25[0-5])|
+	// ([A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}
+	SourceIP        string          `xml:"source_ip"`
+	Count           int             `xml:"count"`
+	PolicyEvaluated PolicyEvaluated `xml:"policy_evaluated"`
+}
+
+type PolicyEvaluated struct {
+	Disposition Disposition            `xml:"disposition"`
+	DKIM        DMARCResult            `xml:"dkim"`
+	SPF         DMARCResult            `xml:"spf"`
+	Reasons     []PolicyOverrideReason `xml:"reason,omitempty"`
+}
+
+// DMARCResult is the final validation and alignment verdict for SPF and DKIM.
+type DMARCResult string
+
+const (
+	DMARCPass DMARCResult = "pass"
+	DMARCFail DMARCResult = "fail"
+)
+
+type PolicyOverrideReason struct {
+	Type    PolicyOverride `xml:"type"`
+	Comment string         `xml:"comment,omitempty"`
+}
+
+// PolicyOverride is a reason the requested DMARC policy from the DNS record
+// was not applied.
+type PolicyOverride string
+
+const (
+	PolicyOverrideForwarded        PolicyOverride = "forwarded"
+	PolicyOverrideSampledOut       PolicyOverride = "sampled_out"
+	PolicyOverrideTrustedForwarder PolicyOverride = "trusted_forwarder"
+	PolicyOverrideMailingList      PolicyOverride = "mailing_list"
+	PolicyOverrideLocalPolicy      PolicyOverride = "local_policy"
+	PolicyOverrideOther            PolicyOverride = "other"
+)
+
+type Identifiers struct {
+	EnvelopeTo   string `xml:"envelope_to,omitempty"`
+	EnvelopeFrom string `xml:"envelope_from"`
+	HeaderFrom   string `xml:"header_from"`
+}
+
+type AuthResults struct {
+	DKIM []DKIMAuthResult `xml:"dkim,omitempty"`
+	SPF  []SPFAuthResult  `xml:"spf"`
+}
+
+type DKIMAuthResult struct {
+	Domain      string     `xml:"domain"`
+	Selector    string     `xml:"selector,omitempty"`
+	Result      DKIMResult `xml:"result"`
+	HumanResult string     `xml:"human_result,omitempty"`
+}
+
+type DKIMResult string
+
+const (
+	DKIMNone      DKIMResult = "none"
+	DKIMPass      DKIMResult = "pass"
+	DKIMFail      DKIMResult = "fail"
+	DKIMPolicy    DKIMResult = "policy"
+	DKIMNeutral   DKIMResult = "neutral"
+	DKIMTemperror DKIMResult = "temperror"
+	DKIMPermerror DKIMResult = "permerror"
+)
+
+type SPFAuthResult struct {
+	Domain string         `xml:"domain"`
+	Scope  SPFDomainScope `xml:"scope"`
+	Result SPFResult      `xml:"result"`
+}
+
+type SPFDomainScope string
+
+const (
+	SPFDomainScopeHelo     SPFDomainScope = "helo"  // SMTP EHLO
+	SPFDomainScopeMailFrom SPFDomainScope = "mfrom" // SMTP "MAIL FROM".
+)
+
+type SPFResult string
+
+const (
+	SPFNone      SPFResult = "none"
+	SPFNeutral   SPFResult = "neutral"
+	SPFPass      SPFResult = "pass"
+	SPFFail      SPFResult = "fail"
+	SPFSoftfail  SPFResult = "softfail"
+	SPFTemperror SPFResult = "temperror"
+	SPFPermerror SPFResult = "permerror"
+)

dmarcrpt/parse.go

@@ -0,0 +1,124 @@
+// Package dmarcrpt parses DMARC aggregate feedback reports.
+package dmarcrpt
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/moxio"
+)
+
+var ErrNoReport = errors.New("no dmarc report found in message")
+
+// ParseReport parses an XML aggregate feedback report.
+// The maximum report size is 20MB.
+func ParseReport(r io.Reader) (*Feedback, error) {
+	r = &moxio.LimitReader{R: r, Limit: 20 * 1024 * 1024}
+	var feedback Feedback
+	d := xml.NewDecoder(r)
+	if err := d.Decode(&feedback); err != nil {
+		return nil, err
+	}
+	return &feedback, nil
+}
+
+// ParseMessageReport parses an aggregate feedback report from a mail message. The
+// maximum message size is 15MB, the maximum report size after decompression is
+// 20MB.
+func ParseMessageReport(r io.ReaderAt) (*Feedback, error) {
+	// ../rfc/7489:1801
+	p, err := message.Parse(&moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024})
+	if err != nil {
+		return nil, fmt.Errorf("parsing mail message: %s", err)
+	}
+
+	return parseMessageReport(p)
+}
+
+func parseMessageReport(p message.Part) (*Feedback, error) {
+	// Pretty much any mime structure is allowed. ../rfc/7489:1861
+	// In practice, some parties will send the report as the only (non-multipart)
+	// content of the message.
+
+	if p.MediaType != "MULTIPART" {
+		return parseReport(p)
+	}
+
+	for {
+		sp, err := p.ParseNextPart()
+		if err == io.EOF {
+			return nil, ErrNoReport
+		}
+		if err != nil {
+			return nil, err
+		}
+		report, err := parseMessageReport(*sp)
+		if err == ErrNoReport {
+			continue
+		} else if err != nil || report != nil {
+			return report, err
+		}
+	}
+}
+
+func parseReport(p message.Part) (*Feedback, error) {
+	ct := strings.ToLower(p.MediaType + "/" + p.MediaSubType)
+	r := p.Reader()
+
+	// If no (useful) content-type is set, try to detect it.
+	if ct == "" || ct == "application/octect-stream" {
+		data := make([]byte, 512)
+		n, err := io.ReadFull(r, data)
+		if err == io.EOF {
+			return nil, ErrNoReport
+		} else if err != nil && err != io.ErrUnexpectedEOF {
+			return nil, fmt.Errorf("reading application/octet-stream for content-type detection: %v", err)
+		}
+		data = data[:n]
+		ct = http.DetectContentType(data)
+		r = io.MultiReader(bytes.NewReader(data), r)
+	}
+
+	switch ct {
+	case "application/zip":
+		// Google sends messages with direct application/zip content-type.
+		return parseZip(r)
+	case "application/gzip":
+		gzr, err := gzip.NewReader(r)
+		if err != nil {
+			return nil, fmt.Errorf("decoding gzip xml report: %s", err)
+		}
+		return ParseReport(gzr)
+	case "text/xml", "application/xml":
+		return ParseReport(r)
+	}
+	return nil, ErrNoReport
+}
+
+func parseZip(r io.Reader) (*Feedback, error) {
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("reading feedback: %s", err)
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
+	if err != nil {
+		return nil, fmt.Errorf("parsing zip file: %s", err)
+	}
+	if len(zr.File) != 1 {
+		return nil, fmt.Errorf("zip contains %d files, expected 1", len(zr.File))
+	}
+	f, err := zr.File[0].Open()
+	if err != nil {
+		return nil, fmt.Errorf("opening file in zip: %s", err)
+	}
+	defer f.Close()
+	return ParseReport(f)
+}

dmarcrpt/parse_test.go

@@ -0,0 +1,179 @@
+package dmarcrpt
+
+import (
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+const reportExample = `<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+  <report_metadata>
+    <org_name>google.com</org_name>
+    <email>noreply-dmarc-support@google.com</email>
+    <extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
+    <report_id>10051505501689795560</report_id>
+    <date_range>
+      <begin>1596412800</begin>
+      <end>1596499199</end>
+    </date_range>
+  </report_metadata>
+  <policy_published>
+    <domain>example.org</domain>
+    <adkim>r</adkim>
+    <aspf>r</aspf>
+    <p>reject</p>
+    <sp>reject</sp>
+    <pct>100</pct>
+  </policy_published>
+  <record>
+    <row>
+      <source_ip>127.0.0.1</source_ip>
+      <count>1</count>
+      <policy_evaluated>
+        <disposition>none</disposition>
+        <dkim>pass</dkim>
+        <spf>pass</spf>
+      </policy_evaluated>
+    </row>
+    <identifiers>
+      <header_from>example.org</header_from>
+    </identifiers>
+    <auth_results>
+      <dkim>
+        <domain>example.org</domain>
+        <result>pass</result>
+        <selector>example</selector>
+      </dkim>
+      <spf>
+        <domain>example.org</domain>
+        <result>pass</result>
+      </spf>
+    </auth_results>
+  </record>
+</feedback>
+`
+
+func TestParseReport(t *testing.T) {
+	var expect = &Feedback{
+		ReportMetadata: ReportMetadata{
+			OrgName:          "google.com",
+			Email:            "noreply-dmarc-support@google.com",
+			ExtraContactInfo: "https://support.google.com/a/answer/2466580",
+			ReportID:         "10051505501689795560",
+			DateRange: DateRange{
+				Begin: 1596412800,
+				End:   1596499199,
+			},
+		},
+		PolicyPublished: PolicyPublished{
+			Domain:          "example.org",
+			ADKIM:           "r",
+			ASPF:            "r",
+			Policy:          "reject",
+			SubdomainPolicy: "reject",
+			Percentage:      100,
+		},
+		Records: []ReportRecord{
+			{
+				Row: Row{
+					SourceIP: "127.0.0.1",
+					Count:    1,
+					PolicyEvaluated: PolicyEvaluated{
+						Disposition: DispositionNone,
+						DKIM:        DMARCPass,
+						SPF:         DMARCPass,
+					},
+				},
+				Identifiers: Identifiers{
+					HeaderFrom: "example.org",
+				},
+				AuthResults: AuthResults{
+					DKIM: []DKIMAuthResult{
+						{
+							Domain:   "example.org",
+							Result:   DKIMPass,
+							Selector: "example",
+						},
+					},
+					SPF: []SPFAuthResult{
+						{
+							Domain: "example.org",
+							Result: SPFPass,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	feedback, err := ParseReport(strings.NewReader(reportExample))
+	if err != nil {
+		t.Fatalf("parsing report: %s", err)
+	}
+	if !reflect.DeepEqual(expect, feedback) {
+		t.Fatalf("expected:\n%#v\ngot:\n%#v", expect, feedback)
+	}
+}
+
+func TestParseMessageReport(t *testing.T) {
+	const dir = "../testdata/dmarc-reports"
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("listing dmarc report emails: %s", err)
+	}
+
+	for _, file := range files {
+		p := dir + "/" + file.Name()
+		f, err := os.Open(p)
+		if err != nil {
+			t.Fatalf("open %q: %s", p, err)
+		}
+		_, err = ParseMessageReport(f)
+		if err != nil {
+			t.Fatalf("ParseMessageReport: %q: %s", p, err)
+		}
+		f.Close()
+	}
+
+	// No report in a non-multipart message.
+	_, err = ParseMessageReport(strings.NewReader("From: <mjl@mox.example>\r\n\r\nNo report.\r\n"))
+	if err != ErrNoReport {
+		t.Fatalf("message without report, got err %#v, expected ErrNoreport", err)
+	}
+
+	// No report in a multipart message.
+	var multipartNoreport = strings.ReplaceAll(`From: <mjl@mox.example>
+To: <mjl@mox.example>
+Subject: Report Domain: mox.example Submitter: mail.mox.example
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="===============5735553800636657282=="
+
+--===============5735553800636657282==
+Content-Type: text/plain
+MIME-Version: 1.0
+
+test
+
+--===============5735553800636657282==
+Content-Type: text/html
+MIME-Version: 1.0
+
+<html></html>
+
+--===============5735553800636657282==--
+`, "\n", "\r\n")
+	_, err = ParseMessageReport(strings.NewReader(multipartNoreport))
+	if err != ErrNoReport {
+		t.Fatalf("message without report, got err %#v, expected ErrNoreport", err)
+	}
+}
+
+func FuzzParseReport(f *testing.F) {
+	f.Add("")
+	f.Add(reportExample)
+	f.Fuzz(func(t *testing.T, s string) {
+		ParseReport(strings.NewReader(s))
+	})
+}

dns/dns.go

@@ -0,0 +1,109 @@
+// Package dns helps parse internationalized domain names (IDNA), canonicalize
+// names and provides a strict and metrics-keeping logging DNS resolver.
+package dns
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"golang.org/x/net/idna"
+)
+
+var errTrailingDot = errors.New("dns name has trailing dot")
+
+// Domain is a domain name, with one or more labels, with at least an ASCII
+// representation, and for IDNA non-ASCII domains a unicode representation.
+// The ASCII string must be used for DNS lookups.
+type Domain struct {
+	// A non-unicode domain, e.g. with A-labels (xn--...) or NR-LDH (non-reserved
+	// letters/digits/hyphens) labels. Always in lower case.
+	ASCII string
+
+	// Name as U-labels. Empty if this is an ASCII-only domain.
+	Unicode string
+}
+
+// Name returns the unicode name if set, otherwise the ASCII name.
+func (d Domain) Name() string {
+	if d.Unicode != "" {
+		return d.Unicode
+	}
+	return d.ASCII
+}
+
+// XName is like Name, but only returns a unicode name when utf8 is true.
+func (d Domain) XName(utf8 bool) string {
+	if utf8 && d.Unicode != "" {
+		return d.Unicode
+	}
+	return d.ASCII
+}
+
+// ASCIIExtra returns the ASCII version of the domain name if smtputf8 is true and
+// this is a unicode domain name. Otherwise it returns an empty string.
+//
+// This function is used to add the punycode name in a comment to SMTP message
+// headers, e.g. Received and Authentication-Results.
+func (d Domain) ASCIIExtra(smtputf8 bool) string {
+	if smtputf8 && d.Unicode != "" {
+		return d.ASCII
+	}
+	return ""
+}
+
+// Strings returns a human-readable string.
+// For IDNA names, the string contains both the unicode and ASCII name.
+func (d Domain) String() string {
+	if d.Unicode == "" {
+		return d.ASCII
+	}
+	return d.Unicode + "/" + d.ASCII
+}
+
+// IsZero returns if this is an empty Domain.
+func (d Domain) IsZero() bool {
+	return d == Domain{}
+}
+
+// ParseDomain parses a domain name that can consist of ASCII-only labels or U
+// labels (unicode).
+// Names are IDN-canonicalized and lower-cased.
+// Characters in unicode can be replaced by equivalents. E.g. "Ⓡ" to "r". This
+// means you should only compare parsed domain names, never strings directly.
+func ParseDomain(s string) (Domain, error) {
+	if strings.HasSuffix(s, ".") {
+		return Domain{}, errTrailingDot
+	}
+	ascii, err := idna.Lookup.ToASCII(s)
+	if err != nil {
+		return Domain{}, fmt.Errorf("to ascii: %w", err)
+	}
+	unicode, err := idna.Lookup.ToUnicode(s)
+	if err != nil {
+		return Domain{}, fmt.Errorf("to unicode: %w", err)
+	}
+	// todo: should we cause errors for unicode domains that were not in
+	// canonical form? we are now accepting all kinds of obscure spellings
+	// for even a basic ASCII domain name.
+	// Also see https://daniel.haxx.se/blog/2022/12/14/idn-is-crazy/
+	if ascii == unicode {
+		return Domain{ascii, ""}, nil
+	}
+	return Domain{ascii, unicode}, nil
+}
+
+// IsNotFound returns whether an error is a net.DNSError with IsNotFound set.
+// IsNotFound means the requested type does not exist for the given domain (a
+// nodata or nxdomain response). It doesn't not necessarily mean no other types
+// for that name exist.
+//
+// A DNS server can respond to a lookup with an error "nxdomain" to indicate a
+// name does not exist (at all), or with a success status with an empty list.
+// The Go resolver returns an IsNotFound error for both cases, there is no need
+// to explicitly check for zero entries.
+func IsNotFound(err error) bool {
+	var dnsErr *net.DNSError
+	return err != nil && errors.As(err, &dnsErr) && dnsErr.IsNotFound
+}

dns/dns_test.go

@@ -0,0 +1,27 @@
+package dns
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestParseDomain(t *testing.T) {
+	test := func(s string, exp Domain, expErr error) {
+		t.Helper()
+		dom, err := ParseDomain(s)
+		if (err == nil) != (expErr == nil) || expErr != nil && !errors.Is(err, expErr) {
+			t.Fatalf("parse domain %q: err %v, expected %v", s, err, expErr)
+		}
+		if expErr == nil && dom != exp {
+			t.Fatalf("parse domain %q: got %#v, epxected %#v", s, dom, exp)
+		}
+	}
+
+	// We rely on normalization of names throughout the code base.
+	test("xmox.nl", Domain{"xmox.nl", ""}, nil)
+	test("XMOX.NL", Domain{"xmox.nl", ""}, nil)
+	test("TEST☺.XMOX.NL", Domain{"xn--test-3o3b.xmox.nl", "test☺.xmox.nl"}, nil)
+	test("TEST☺.XMOX.NL", Domain{"xn--test-3o3b.xmox.nl", "test☺.xmox.nl"}, nil)
+	test("ℂᵤⓇℒ。𝐒🄴", Domain{"curl.se", ""}, nil) // https://daniel.haxx.se/blog/2022/12/14/idn-is-crazy/
+	test("xmox.nl.", Domain{}, errTrailingDot)
+}

dns/ipdomain.go

@@ -0,0 +1,42 @@
+package dns
+
+import (
+	"net"
+)
+
+// IPDomain is an ip address, a domain, or empty.
+type IPDomain struct {
+	IP     net.IP
+	Domain Domain
+}
+
+// IsZero returns if both IP and Domain are zero.
+func (d IPDomain) IsZero() bool {
+	return d.IP == nil && d.Domain == Domain{}
+}
+
+// String returns a string representation of either the IP or domain (with
+// UTF-8).
+func (d IPDomain) String() string {
+	if len(d.IP) > 0 {
+		return d.IP.String()
+	}
+	return d.Domain.Name()
+}
+
+// XString is like String, but only returns UTF-8 domains if utf8 is true.
+func (d IPDomain) XString(utf8 bool) string {
+	if d.IsIP() {
+		// todo: check callers if this is valid syntax for them. should we add [] for ipv6? perhaps also ipv4? probably depends on context. in smtp, the syntax is [<ipv4>] and [IPv6:<ipv6>].
+		return d.IP.String()
+	}
+	return d.Domain.XName(utf8)
+}
+
+func (d IPDomain) IsIP() bool {
+	return len(d.IP) > 0
+}
+
+func (d IPDomain) IsDomain() bool {
+	return !d.Domain.IsZero()
+}

dns/mock.go

@@ -0,0 +1,156 @@
+package dns
+
+import (
+	"context"
+	"fmt"
+	"net"
+)
+
+// MockResolver is a Resolver used for testing.
+// Set DNS records in the fields, which map FQDNs (with trailing dot) to values.
+type MockResolver struct {
+	PTR   map[string][]string
+	A     map[string][]string
+	AAAA  map[string][]string
+	TXT   map[string][]string
+	MX    map[string][]*net.MX
+	CNAME map[string]string
+	Fail  map[Mockreq]struct{}
+}
+
+type Mockreq struct {
+	Type string // E.g. "cname", "txt", "mx", "ptr", etc.
+	Name string
+}
+
+var _ Resolver = MockResolver{}
+
+func (r MockResolver) nxdomain(s string) *net.DNSError {
+	return &net.DNSError{
+		Err:        "no record",
+		Name:       s,
+		Server:     "localhost",
+		IsNotFound: true,
+	}
+}
+
+func (r MockResolver) servfail(s string) *net.DNSError {
+	return &net.DNSError{
+		Err:         "temp error",
+		Name:        s,
+		Server:      "localhost",
+		IsTemporary: true,
+	}
+}
+
+func (r MockResolver) LookupCNAME(ctx context.Context, name string) (string, error) {
+	if _, ok := r.Fail[Mockreq{"cname", name}]; ok {
+		return "", r.servfail(name)
+	}
+	if cname, ok := r.CNAME[name]; ok {
+		return cname, nil
+	}
+	return "", r.nxdomain(name)
+}
+
+func (r MockResolver) LookupAddr(ctx context.Context, ip string) ([]string, error) {
+	if _, ok := r.Fail[Mockreq{"ptr", ip}]; ok {
+		return nil, r.servfail(ip)
+	}
+	l, ok := r.PTR[ip]
+	if !ok {
+		return nil, r.nxdomain(ip)
+	}
+	return l, nil
+}
+
+func (r MockResolver) LookupNS(ctx context.Context, name string) ([]*net.NS, error) {
+	return nil, r.servfail("ns not implemented")
+}
+
+func (r MockResolver) LookupPort(ctx context.Context, network, service string) (port int, err error) {
+	return 0, r.servfail("port not implemented")
+}
+
+func (r MockResolver) LookupSRV(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) {
+	return "", nil, r.servfail("srv not implemented")
+}
+
+func (r MockResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) {
+	if _, ok := r.Fail[Mockreq{"ipaddr", host}]; ok {
+		return nil, r.servfail(host)
+	}
+	addrs, err := r.LookupHost(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	ips := make([]net.IPAddr, len(addrs))
+	for i, a := range addrs {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			return nil, fmt.Errorf("malformed ip %q", a)
+		}
+		ips[i] = net.IPAddr{IP: ip}
+	}
+	return ips, nil
+}
+
+func (r MockResolver) LookupHost(ctx context.Context, host string) (addrs []string, err error) {
+	if _, ok := r.Fail[Mockreq{"host", host}]; ok {
+		return nil, r.servfail(host)
+	}
+	addrs = append(addrs, r.A[host]...)
+	addrs = append(addrs, r.AAAA[host]...)
+	if len(addrs) > 0 {
+		return addrs, nil
+	}
+	if cname, ok := r.CNAME[host]; ok {
+		return []string{cname}, nil
+	}
+	return nil, r.nxdomain(host)
+}
+
+func (r MockResolver) LookupIP(ctx context.Context, network, host string) ([]net.IP, error) {
+	if _, ok := r.Fail[Mockreq{"ip", host}]; ok {
+		return nil, r.servfail(host)
+	}
+	var ips []net.IP
+	switch network {
+	case "ip", "ip4":
+		for _, ip := range r.A[host] {
+			ips = append(ips, net.ParseIP(ip))
+		}
+	}
+	switch network {
+	case "ip", "ip6":
+		for _, ip := range r.AAAA[host] {
+			ips = append(ips, net.ParseIP(ip))
+		}
+	}
+	if len(ips) == 0 {
+		return nil, r.nxdomain(host)
+	}
+	return ips, nil
+}
+
+func (r MockResolver) LookupMX(ctx context.Context, name string) ([]*net.MX, error) {
+	if _, ok := r.Fail[Mockreq{"mx", name}]; ok {
+		return nil, r.servfail(name)
+	}
+	l, ok := r.MX[name]
+	if !ok {
+		return nil, r.nxdomain(name)
+	}
+	return l, nil
+}
+
+func (r MockResolver) LookupTXT(ctx context.Context, name string) ([]string, error) {
+	if _, ok := r.Fail[Mockreq{"txt", name}]; ok {
+		return nil, r.servfail(name)
+	}
+	l, ok := r.TXT[name]
+	if !ok {
+		return nil, r.nxdomain(name)
+	}
+	return l, nil
+}

dns/resolver.go

@@ -0,0 +1,248 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+// todo future: replace with a dnssec capable resolver
+// todo future: change to interface that is closer to DNS. 1. expose nxdomain vs success with zero entries: nxdomain means the name does not exist for any dns resource record type, success with zero records means the name exists for other types than the requested type; 2. add ability to not follow cname records when resolving. the net resolver automatically follows cnames for LookupHost, LookupIP, LookupIPAddr. when resolving names found in mx records, we explicitly must not follow cnames. that seems impossible at the moment. 3. when looking up a cname, actually lookup the record? "net" LookupCNAME will return the requested name with no error if there is no CNAME record. because it returns the canonical name.
+// todo future: add option to not use anything in the cache, for the admin pages where you check the latest DNS settings, ignoring old cached info.
+
+var xlog = mlog.New("dns")
+
+var (
+	metricLookup = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "mox_dns_lookup_duration_seconds",
+			Help:    "DNS lookups.",
+			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
+		},
+		[]string{
+			"pkg",
+			"type",   // Lower-case Resolver method name without leading Lookup.
+			"result", // ok, nxdomain, temporary, timeout, canceled, error
+		},
+	)
+)
+
+// Resolver is the interface strict resolver implements.
+type Resolver interface {
+	LookupAddr(ctx context.Context, addr string) ([]string, error)
+	LookupCNAME(ctx context.Context, host string) (string, error) // NOTE: returns an error if no CNAME record is present.
+	LookupHost(ctx context.Context, host string) (addrs []string, err error)
+	LookupIP(ctx context.Context, network, host string) ([]net.IP, error)
+	LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
+	LookupMX(ctx context.Context, name string) ([]*net.MX, error)
+	LookupNS(ctx context.Context, name string) ([]*net.NS, error)
+	LookupPort(ctx context.Context, network, service string) (port int, err error)
+	LookupSRV(ctx context.Context, service, proto, name string) (string, []*net.SRV, error)
+	LookupTXT(ctx context.Context, name string) ([]string, error)
+}
+
+// WithPackage sets Pkg on resolver if it is a StrictResolve and does not have a package set yet.
+func WithPackage(resolver Resolver, name string) Resolver {
+	r, ok := resolver.(StrictResolver)
+	if ok && r.Pkg == "" {
+		nr := r
+		r.Pkg = name
+		return nr
+	}
+	return resolver
+}
+
+// StrictResolver is a net.Resolver that enforces that DNS names end with a dot,
+// preventing "search"-relative lookups.
+type StrictResolver struct {
+	Pkg      string        // Name of subsystem that is making DNS requests, for metrics.
+	Resolver *net.Resolver // Where the actual lookups are done. If nil, net.DefaultResolver is used for lookups.
+}
+
+var _ Resolver = StrictResolver{}
+
+var ErrRelativeDNSName = errors.New("dns: host to lookup must be absolute, ending with a dot")
+
+func metricLookupObserve(pkg, typ string, err error, start time.Time) {
+	var result string
+	var dnsErr *net.DNSError
+	switch {
+	case err == nil:
+		result = "ok"
+	case errors.As(err, &dnsErr) && dnsErr.IsNotFound:
+		result = "nxdomain"
+	case errors.As(err, &dnsErr) && dnsErr.IsTemporary:
+		result = "temporary"
+	case errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) || errors.As(err, &dnsErr) && dnsErr.IsTimeout:
+		result = "timeout"
+	case errors.Is(err, context.Canceled):
+		result = "canceled"
+	default:
+		result = "error"
+	}
+	metricLookup.WithLabelValues(pkg, typ, result).Observe(float64(time.Since(start)) / float64(time.Second))
+}
+
+func (r StrictResolver) WithPackage(name string) Resolver {
+	nr := r
+	nr.Pkg = name
+	return nr
+}
+
+func (r StrictResolver) resolver() Resolver {
+	if r.Resolver == nil {
+		return net.DefaultResolver
+	}
+	return r.Resolver
+}
+
+func (r StrictResolver) LookupAddr(ctx context.Context, addr string) (resp []string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "addr", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "addr"), mlog.Field("addr", addr), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	resp, err = r.resolver().LookupAddr(ctx, addr)
+	return
+}
+
+// LookupCNAME looks up a CNAME. Unlike "net" LookupCNAME, it returns a "not found"
+// error if there is no CNAME record.
+func (r StrictResolver) LookupCNAME(ctx context.Context, host string) (resp string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "cname", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "cname"), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return "", ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupCNAME(ctx, host)
+	if err == nil && resp == host {
+		return "", &net.DNSError{
+			Err:        "no cname record",
+			Name:       host,
+			Server:     "",
+			IsNotFound: true,
+		}
+	}
+	return
+}
+func (r StrictResolver) LookupHost(ctx context.Context, host string) (resp []string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "host", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "host"), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupHost(ctx, host)
+	return
+}
+
+func (r StrictResolver) LookupIP(ctx context.Context, network, host string) (resp []net.IP, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "ip", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "ip"), mlog.Field("network", network), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupIP(ctx, network, host)
+	return
+}
+
+func (r StrictResolver) LookupIPAddr(ctx context.Context, host string) (resp []net.IPAddr, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "ipaddr", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "ipaddr"), mlog.Field("host", host), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(host, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupIPAddr(ctx, host)
+	return
+}
+
+func (r StrictResolver) LookupMX(ctx context.Context, name string) (resp []*net.MX, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "mx", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "mx"), mlog.Field("name", name), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupMX(ctx, name)
+	return
+}
+
+func (r StrictResolver) LookupNS(ctx context.Context, name string) (resp []*net.NS, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "ns", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "ns"), mlog.Field("name", name), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupNS(ctx, name)
+	return
+}
+
+func (r StrictResolver) LookupPort(ctx context.Context, network, service string) (resp int, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "port", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "port"), mlog.Field("network", network), mlog.Field("service", service), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	resp, err = r.resolver().LookupPort(ctx, network, service)
+	return
+}
+
+func (r StrictResolver) LookupSRV(ctx context.Context, service, proto, name string) (resp0 string, resp1 []*net.SRV, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "srv", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "srv"), mlog.Field("service", service), mlog.Field("proto", proto), mlog.Field("name", name), mlog.Field("resp0", resp0), mlog.Field("resp1", resp1), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return "", nil, ErrRelativeDNSName
+	}
+	resp0, resp1, err = r.resolver().LookupSRV(ctx, service, proto, name)
+	return
+}
+
+func (r StrictResolver) LookupTXT(ctx context.Context, name string) (resp []string, err error) {
+	start := time.Now()
+	defer func() {
+		metricLookupObserve(r.Pkg, "txt", err, start)
+		xlog.WithContext(ctx).Debugx("dns lookup result", err, mlog.Field("pkg", r.Pkg), mlog.Field("type", "txt"), mlog.Field("name", name), mlog.Field("resp", resp), mlog.Field("duration", time.Since(start)))
+	}()
+
+	if !strings.HasSuffix(name, ".") {
+		return nil, ErrRelativeDNSName
+	}
+	resp, err = r.resolver().LookupTXT(ctx, name)
+	return
+}

dnsbl/dnsbl.go

@@ -0,0 +1,130 @@
+// Package dnsbl implements DNS block lists (RFC 5782), for checking incoming messages from sources without reputation.
+package dnsbl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+)
+
+var xlog = mlog.New("dnsbl")
+
+var (
+	metricLookup = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "mox_dnsbl_lookup_duration_seconds",
+			Help:    "DNSBL lookup",
+			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
+		},
+		[]string{
+			"zone",
+			"status",
+		},
+	)
+)
+
+var ErrDNS = errors.New("dnsbl: dns error")
+
+// Status is the result of a DNSBL lookup.
+type Status string
+
+var (
+	StatusTemperr Status = "temperror" // Temporary failure.
+	StatusPass    Status = "pass"      // Not present in block list.
+	StatusFail    Status = "fail"      // Present in block list.
+)
+
+// Lookup checks if "ip" occurs in the DNS block list "zone" (e.g. dnsbl.example.org).
+func Lookup(ctx context.Context, resolver dns.Resolver, zone dns.Domain, ip net.IP) (rstatus Status, rexplanation string, rerr error) {
+	log := xlog.WithContext(ctx)
+	start := time.Now()
+	defer func() {
+		metricLookup.WithLabelValues(zone.Name(), string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second))
+		log.Debugx("dnsbl lookup result", rerr, mlog.Field("zone", zone), mlog.Field("ip", ip), mlog.Field("status", rstatus), mlog.Field("explanation", rexplanation), mlog.Field("duration", time.Since(start)))
+	}()
+
+	b := &strings.Builder{}
+	v4 := ip.To4()
+	if v4 != nil {
+		// ../rfc/5782:148
+		s := len(v4) - 1
+		for i := s; i >= 0; i-- {
+			if i < s {
+				b.WriteByte('.')
+			}
+			b.WriteString(strconv.Itoa(int(v4[i])))
+		}
+	} else {
+		// ../rfc/5782:270
+		s := len(ip) - 1
+		const chars = "0123456789abcdef"
+		for i := s; i >= 0; i-- {
+			if i < s {
+				b.WriteByte('.')
+			}
+			v := ip[i]
+			b.WriteByte(chars[v>>0&0xf])
+			b.WriteByte('.')
+			b.WriteByte(chars[v>>4&0xf])
+		}
+	}
+	b.WriteString("." + zone.ASCII + ".")
+	addr := b.String()
+
+	// ../rfc/5782:175
+	_, err := dns.WithPackage(resolver, "dnsbl").LookupIP(ctx, "ip4", addr)
+	if dns.IsNotFound(err) {
+		return StatusPass, "", nil
+	} else if err != nil {
+		return StatusTemperr, "", fmt.Errorf("%w: %s", ErrDNS, err)
+	}
+
+	txts, err := dns.WithPackage(resolver, "dnsbl").LookupTXT(ctx, addr)
+	if dns.IsNotFound(err) {
+		return StatusFail, "", nil
+	} else if err != nil {
+		log.Debugx("looking up txt record from dnsbl", err, mlog.Field("addr", addr))
+		return StatusFail, "", nil
+	}
+	return StatusFail, strings.Join(txts, "; "), nil
+}
+
+// CheckHealth checks whether the DNSBL "zone" is operating correctly by
+// querying for 127.0.0.2 (must be present) and 127.0.0.1 (must not be present).
+// Users of a DNSBL should periodically check if the DNSBL is still operating
+// properly.
+// For temporary errors, ErrDNS is returned.
+func CheckHealth(ctx context.Context, resolver dns.Resolver, zone dns.Domain) (rerr error) {
+	log := xlog.WithContext(ctx)
+	start := time.Now()
+	defer func() {
+		log.Debugx("dnsbl healthcheck result", rerr, mlog.Field("zone", zone), mlog.Field("duration", time.Since(start)))
+	}()
+
+	// ../rfc/5782:355
+	status1, _, err1 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 1))
+	status2, _, err2 := Lookup(ctx, resolver, zone, net.IPv4(127, 0, 0, 2))
+	if status1 == StatusPass && status2 == StatusFail {
+		return nil
+	} else if status1 == StatusFail {
+		return fmt.Errorf("dnsbl contains unwanted test address 127.0.0.1")
+	} else if status2 == StatusPass {
+		return fmt.Errorf("dnsbl does not contain required test address 127.0.0.2")
+	}
+	if err1 != nil {
+		return err1
+	} else if err2 != nil {
+		return err2
+	}
+	return ErrDNS
+}

dnsbl/dnsbl_test.go

@@ -0,0 +1,64 @@
+package dnsbl
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/mjl-/mox/dns"
+)
+
+func TestDNSBL(t *testing.T) {
+	ctx := context.Background()
+
+	resolver := dns.MockResolver{
+		A: map[string][]string{
+			"2.0.0.127.example.com.": {"127.0.0.2"}, // required for health
+			"1.0.0.10.example.com.":  {"127.0.0.2"},
+			"b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.example.com.": {"127.0.0.2"},
+		},
+		TXT: map[string][]string{
+			"1.0.0.10.example.com.": {"listed!"},
+			"b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.example.com.": {"listed!"},
+		},
+	}
+
+	if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.1")); err != nil {
+		t.Fatalf("lookup: %v", err)
+	} else if status != StatusFail {
+		t.Fatalf("lookup, got status %v, expected fail", status)
+	} else if expl != "listed!" {
+		t.Fatalf("lookup, got explanation %q", expl)
+	}
+
+	if status, expl, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("2001:db8:1:2:3:4:567:89ab")); err != nil {
+		t.Fatalf("lookup: %v", err)
+	} else if status != StatusFail {
+		t.Fatalf("lookup, got status %v, expected fail", status)
+	} else if expl != "listed!" {
+		t.Fatalf("lookup, got explanation %q", expl)
+	}
+
+	if status, _, err := Lookup(ctx, resolver, dns.Domain{ASCII: "example.com"}, net.ParseIP("10.0.0.2")); err != nil {
+		t.Fatalf("lookup: %v", err)
+	} else if status != StatusPass {
+		t.Fatalf("lookup, got status %v, expected pass", status)
+	}
+
+	// ../rfc/5782:357
+	if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.com"}); err != nil {
+		t.Fatalf("dnsbl not healthy: %v", err)
+	}
+	if err := CheckHealth(ctx, resolver, dns.Domain{ASCII: "example.org"}); err == nil {
+		t.Fatalf("bad dnsbl is healthy")
+	}
+
+	unhealthyResolver := dns.MockResolver{
+		A: map[string][]string{
+			"1.0.0.127.example.com.": {"127.0.0.2"}, // Should not be present in healthy dnsbl.
+		},
+	}
+	if err := CheckHealth(ctx, unhealthyResolver, dns.Domain{ASCII: "example.com"}); err == nil {
+		t.Fatalf("bad dnsbl is healthy")
+	}
+}

doc.go

@@ -0,0 +1,613 @@
+/*
+Command mox is a modern full-featured open source secure mail server for
+low-maintenance self-hosted email.
+
+  - Quick and easy to set up with quickstart and automatic TLS with ACME and
+    Let's Encrypt.
+  - IMAP4 with extensions for accessing email.
+  - SMTP with SPF, DKIM, DMARC, DNSBL, MTA-STS, TLSRPT for exchanging email.
+  - Reputation-based and content-based spam filtering.
+  - Internationalized email.
+  - Admin web interface.
+
+# Commands
+
+	mox [-config mox.conf] ...
+	mox serve
+	mox quickstart user@domain
+	mox restart
+	mox stop
+	mox setaccountpassword address
+	mox setadminpassword
+	mox loglevels [level [pkg]]
+	mox queue list
+	mox queue kick [-id id] [-todomain domain] [-recipient address]
+	mox queue drop [-id id] [-todomain domain] [-recipient address]
+	mox queue dump id
+	mox import maildir accountname mailboxname maildir
+	mox import mbox accountname mailboxname mbox
+	mox export maildir dst-path account-path [mailbox]
+	mox export mbox dst-path account-path [mailbox]
+	mox help [command ...]
+	mox config test
+	mox config dnscheck domain
+	mox config dnsrecords domain
+	mox config describe-domains >domains.conf
+	mox config describe-static >mox.conf
+	mox config account add account address
+	mox config account rm account
+	mox config address add address account
+	mox config address rm address
+	mox config domain add domain account [localpart]
+	mox config domain rm domain
+	mox checkupdate
+	mox cid cid
+	mox clientconfig domain
+	mox dkim gened25519 >$selector._domainkey.$domain.ed25519key.pkcs8.pem
+	mox dkim genrsa >$selector._domainkey.$domain.rsakey.pkcs8.pem
+	mox dkim lookup selector domain
+	mox dkim txt <$selector._domainkey.$domain.key.pkcs8.pem
+	mox dkim verify message
+	mox dmarc lookup domain
+	mox dmarc parsereportmsg message ...
+	mox dmarc verify remoteip mailfromaddress helodomain < message
+	mox dnsbl check zone ip
+	mox dnsbl checkhealth zone
+	mox mtasts lookup domain
+	mox sendmail [-Fname] [ignoredflags] [-t] [<message]
+	mox spf check domain ip
+	mox spf lookup domain
+	mox spf parse txtrecord
+	mox tlsrpt lookup domain
+	mox tlsrpt parsereportmsg message ...
+	mox version
+
+Many commands talk to a running mox instance, through the ctl file in the data
+directory. Specify the configuration file (that holds the path to the data
+directory) through the -config flag or MOXCONF environment variable.
+
+# mox serve
+
+Start mox, serving SMTP/IMAP/HTTPS.
+
+Incoming email is accepted over SMTP. Email can be retrieved by users using
+IMAP. HTTP listeners are started for the admin/account web interfaces, and for
+automated TLS configuration. Missing essential TLS certificates are immediately
+requested, other TLS certificates are requested on demand.
+
+	usage: mox serve
+
+# mox quickstart
+
+Quickstart generates configuration files and prints instructions to quickly set up a mox instance.
+
+Quickstart prints initial admin and account passwords, configuration files, DNS
+records you should create, instructions for setting correct user/group and
+permissions, and if you run it on Linux it prints a systemd service file.
+
+	usage: mox quickstart user@domain
+
+# mox restart
+
+Restart mox after validating the configuration file.
+
+Restart execs the mox binary, which have been updated. Restart returns after
+the restart has finished. If you update the mox binary, keep in mind that the
+validation of the configuration file is done by the old process with the old
+binary. The new binary may report a syntax error. If you update the binary, you
+should use the "config test" command with the new binary to validate the
+configuration file.
+
+Like stop, existing connections get a 3 second period for graceful shutdown.
+
+	usage: mox restart
+
+# mox stop
+
+Shut mox down, giving connections maximum 3 seconds to stop before closing them.
+
+While shutting down, new IMAP and SMTP connections will get a status response
+indicating temporary unavailability. Existing connections will get a 3 second
+period to finish their transaction and shut down. Under normal circumstances,
+only IMAP has long-living connections, with the IDLE command to get notified of
+new mail deliveries.
+
+	usage: mox stop
+
+# mox setaccountpassword
+
+Set new password an account.
+
+The password is read from stdin. Its bcrypt hash and SCRAM-SHA-256 derivations
+are stored in the accounts database.
+
+Any email address configured for the account can be used.
+
+	usage: mox setaccountpassword address
+
+# mox setadminpassword
+
+Set a new admin password, for the web interface.
+
+The password is read from stdin. Its bcrypt hash is stored in a file named
+"adminpasswd" in the configuration directory.
+
+	usage: mox setadminpassword
+
+# mox loglevels
+
+Print the log levels, or set a new default log level, or a level for the given package.
+
+By default, a single log level applies to all logging in mox. But for each
+"pkg", an overriding log level can be configured. Examples of packages:
+smtpserver, smtpclient, queue, imapserver, spf, dkim, dmarc, junk, message,
+etc.
+
+Valid labels: error, info, debug, trace.
+
+	usage: mox loglevels [level [pkg]]
+
+# mox queue list
+
+List messages in the delivery queue.
+
+This prints the message with its ID, last and next delivery attempts, last
+error.
+
+	usage: mox queue list
+
+# mox queue kick
+
+Schedule matching messages in the queue for immediate delivery.
+
+Messages deliveries are normally attempted with exponential backoff. The first
+retry after 7.5 minutes, and doubling each time. Kicking messages sets their
+next scheduled attempt to now, it can cause delivery to fail earlier than
+without rescheduling.
+
+	usage: mox queue kick [-id id] [-todomain domain] [-recipient address]
+	  -id int
+	    	id of message in queue
+	  -recipient string
+	    	recipient email address
+	  -todomain string
+	    	destination domain of messages
+
+# mox queue drop
+
+Remove matching messages from the queue.
+
+Dangerous operation, this completely removes the message. If you want to store
+the message, use "queue dump" before removing.
+
+	usage: mox queue drop [-id id] [-todomain domain] [-recipient address]
+	  -id int
+	    	id of message in queue
+	  -recipient string
+	    	recipient email address
+	  -todomain string
+	    	destination domain of messages
+
+# mox queue dump
+
+Dump a message from the queue.
+
+The message is printed to stdout and is in standard internet mail format.
+
+	usage: mox queue dump id
+
+# mox import maildir
+
+Import a maildir into an account.
+
+By default, messages will train the junk filter based on their flags and
+mailbox naming. If the destination mailbox name starts with "junk" or "spam"
+(case insensitive), messages are imported and trained as junk regardless of
+pre-existing flags. Use the -train=false flag to prevent training the filter.
+
+If the destination mailbox is "Sent", the recipients of the messages are added
+to the message metadata, causing later incoming messages from these recipients
+to be accepted, unless other reputation signals prevent that.
+
+The message "read"/"seen" flag can be overridden during import with the
+-markread flag.
+
+Mailbox flags, like "seen", "answered", "forwarded", will be imported. An
+attempt is made to parse dovecot keyword files.
+
+The maildir files/directories are read by the mox process, so make sure it has
+access to the maildir directories/files.
+
+	usage: mox import maildir accountname mailboxname maildir
+	  -markread
+	    	mark all imported messages as read
+	  -train
+	    	train junkfilter with messages (default true)
+
+# mox import mbox
+
+Import an mbox into an account.
+
+Using mbox is not recommended, maildir is a better format.
+
+By default, messages will train the junk filter based on their flags and
+mailbox naming. If the destination mailbox name starts with "junk" or "spam"
+(case insensitive), messages are imported and trained as junk regardless of
+pre-existing flags. Use the -train=false flag to prevent training the filter.
+
+If the destination mailbox is "Sent", the recipients of the messages are added
+to the message metadata, causing later incoming messages from these recipients
+to be accepted, unless other reputation signals prevent that.
+
+The message "read"/"seen" flag can be overridden during import with the
+-markread flag.
+
+The mailbox is read by the mox process, so make sure it has access to the
+maildir directories/files.
+
+	usage: mox import mbox accountname mailboxname mbox
+	  -markread
+	    	mark all imported messages as read
+	  -train
+	    	train junkfilter with messages (default true)
+
+# mox export maildir
+
+Export one or all mailboxes from an account in maildir format.
+
+Export bypasses a running mox instance. It opens the account mailbox/message
+database file directly. This may block if a running mox instance also has the
+database open, e.g. for IMAP connections.
+
+	usage: mox export maildir dst-path account-path [mailbox]
+
+# mox export mbox
+
+Export messages from one or all mailboxes in an account in mbox format.
+
+Using mbox is not recommended. Maildir is a better format.
+
+Export bypasses a running mox instance. It opens the account mailbox/message
+database file directly. This may block if a running mox instance also has the
+database open, e.g. for IMAP connections.
+
+For mbox export, we use "mboxrd" where message lines starting with the magic
+"From " string are escaped by prepending a >. We escape all ">*From ",
+otherwise reconstructing the original could lose a ">".
+
+	usage: mox export mbox dst-path account-path [mailbox]
+
+# mox help
+
+Prints help about matching commands.
+
+If multiple commands match, they are listed along with the first line of their help text.
+If a single command matches, its usage and full help text is printed.
+
+	usage: mox help [command ...]
+
+# mox config test
+
+Parses and validates the configuration files.
+
+If valid, the command exits with status 0. If not valid, all errors encountered
+are printed.
+
+	usage: mox config test
+
+# mox config dnscheck
+
+Check the DNS records with the configuration for the domain, and print any errors/warnings.
+
+	usage: mox config dnscheck domain
+
+# mox config dnsrecords
+
+Prints annotated DNS records as zone file that should be created for the domain.
+
+The zone file can be imported into existing DNS software. You should review the
+DNS records, especially if your domain previously/currently has email
+configured.
+
+	usage: mox config dnsrecords domain
+
+# mox config describe-domains
+
+Prints an annotated empty configuration for use as domains.conf.
+
+The domains configuration file contains the domains and their configuration,
+and accounts and their configuration. This includes the configured email
+addresses. The mox admin web interface, and the mox command line interface, can
+make changes to this file. Mox automatically reloads this file when it changes.
+
+Like the static configuration, the example domains.conf printed by this command
+needs modifications to make it valid.
+
+	usage: mox config describe-domains >domains.conf
+
+# mox config describe-static
+
+Prints an annotated empty configuration for use as mox.conf.
+
+The static configuration file cannot be reloaded while mox is running. Mox has
+to be restarted for changes to the static configuration file to take effect.
+
+This configuration file needs modifications to make it valid. For example, it
+may contain unfinished list items.
+
+	usage: mox config describe-static >mox.conf
+
+# mox config account add
+
+Add an account with an email address and reload the configuration.
+
+Email can be delivered to this address/account. A password has to be configured
+explicitly, see the setaccountpassword command.
+
+	usage: mox config account add account address
+
+# mox config account rm
+
+Remove an account and reload the configuration.
+
+Email addresses for this account will also be removed, and incoming email for
+these addresses will be rejected.
+
+	usage: mox config account rm account
+
+# mox config address add
+
+Adds an address to an account and reloads the configuration.
+
+	usage: mox config address add address account
+
+# mox config address rm
+
+Remove an address and reload the configuration.
+
+Incoming email for this address will be rejected.
+
+	usage: mox config address rm address
+
+# mox config domain add
+
+Adds a new domain to the configuration and reloads the configuration.
+
+The account is used for the postmaster mailboxes the domain, including as DMARC and
+TLS reporting. Localpart is the "username" at the domain for this account. If
+must be set if and only if account does not yet exist.
+
+	usage: mox config domain add domain account [localpart]
+
+# mox config domain rm
+
+Remove a domain from the configuration and reload the configuration.
+
+This is a dangerous operation. Incoming email delivery for this domain will be
+rejected.
+
+	usage: mox config domain rm domain
+
+# mox checkupdate
+
+Check if a newer version of mox is available.
+
+A single DNS TXT lookup to _updates.xmox.nl tells if a new version is
+available. If so, a changelog is fetched from https://updates.xmox.nl, and the
+individual entries validated with a builtin public key. The changelog is
+printed.
+
+	usage: mox checkupdate
+
+# mox cid
+
+Turn an ID from a Received header into a cid, for looking up in logs.
+
+A cid is essentially a connection counter initialized when mox starts. Each log
+line contains a cid. Received headers added by mox contain a unique ID that can
+be decrypted to a cid by admin of a mox instance only.
+
+	usage: mox cid cid
+
+# mox clientconfig
+
+Print the configuration for email clients for a domain.
+
+Sending email is typically not done on the SMTP port 25, but on submission
+ports 465 (with TLS) and 587 (without initial TLS, but usually added to the
+connection with STARTTLS). For IMAP, the port with TLS is 993 and without is
+143.
+
+Without TLS/STARTTLS, passwords are sent in clear text, which should only be
+configured over otherwise secured connections, like a VPN.
+
+	usage: mox clientconfig domain
+
+# mox dkim gened25519
+
+Generate a new ed25519 key for use with DKIM.
+
+Ed25519 keys are much smaller than RSA keys of comparable cryptographic
+strength. This is convenient because of maximum DNS message sizes. At the time
+of writing, not many mail servers appear to support ed25519 DKIM keys though,
+so it is recommended to sign messages with both RSA and ed25519 keys.
+
+	usage: mox dkim gened25519 >$selector._domainkey.$domain.ed25519key.pkcs8.pem
+
+# mox dkim genrsa
+
+Generate a new 2048 bit RSA private key for use with DKIM.
+
+The generated file is in PEM format, and has a comment it is generated for use
+with DKIM, by mox.
+
+	usage: mox dkim genrsa >$selector._domainkey.$domain.rsakey.pkcs8.pem
+
+# mox dkim lookup
+
+Lookup and print the DKIM record for the selector at the domain.
+
+	usage: mox dkim lookup selector domain
+
+# mox dkim txt
+
+Print a DKIM DNS TXT record with the public key derived from the private key read from stdin.
+
+The DNS should be configured as a TXT record at $selector._domainkey.$domain.
+
+	usage: mox dkim txt <$selector._domainkey.$domain.key.pkcs8.pem
+
+# mox dkim verify
+
+Verify the DKIM signatures in a message and print the results.
+
+The message is parsed, and the DKIM-Signature headers are validated. Validation
+of older messages may fail because the DNS records have been removed or changed
+by now, or because the signature header may have specified an expiration time
+that was passed.
+
+	usage: mox dkim verify message
+
+# mox dmarc lookup
+
+Lookup dmarc policy for domain, a DNS TXT record at _dmarc.<domain>, validate and print it.
+
+	usage: mox dmarc lookup domain
+
+# mox dmarc parsereportmsg
+
+Parse a DMARC report from an email message, and print its extracted details.
+
+DMARC reports are periodically mailed, if requested in the DMARC DNS record of
+a domain. Reports are sent by mail servers that received messages with our
+domain in a From header. This may or may not be legatimate email. DMARC reports
+contain summaries of evaluations of DMARC and DKIM/SPF, which can help
+understand email deliverability problems.
+
+	usage: mox dmarc parsereportmsg message ...
+
+# mox dmarc verify
+
+Parse an email message and evaluate it against the DMARC policy of the domain in the From-header.
+
+mailfromaddress and helodomain are used for SPF validation. If both are empty,
+SPF validation is skipped.
+
+mailfromaddress should be the address used as MAIL FROM in the SMTP session.
+For DSN messages, that address may be empty. The helo domain was specified at
+the beginning of the SMTP transaction that delivered the message. These values
+can be found in message headers.
+
+	usage: mox dmarc verify remoteip mailfromaddress helodomain < message
+
+# mox dnsbl check
+
+Test if IP is in the DNS blocklist of the zone, e.g. bl.spamcop.net.
+
+If the IP is in the blocklist, an explanation is printed. This is typically a
+URL with more information.
+
+	usage: mox dnsbl check zone ip
+
+# mox dnsbl checkhealth
+
+Check the health of the DNS blocklist represented by zone, e.g. bl.spamcop.net.
+
+The health of a DNS blocklist can be checked by querying for 127.0.0.1 and
+127.0.0.2. The second must and the first must not be present.
+
+	usage: mox dnsbl checkhealth zone
+
+# mox mtasts lookup
+
+Lookup the MTASTS record and policy for the domain.
+
+MTA-STS is a mechanism for a domain to specify if it requires TLS connections
+for delivering email. If a domain has a valid MTA-STS DNS TXT record at
+_mta-sts.<domain> it signals it implements MTA-STS. A policy can then be
+fetched at https://mta-sts.<domain>/.well-known/mta-sts.txt. The policy
+specifies the mode (enforce, testing, none), which MX servers support TLS and
+should be used, and how long the policy can be cached.
+
+	usage: mox mtasts lookup domain
+
+# mox sendmail
+
+Sendmail is a drop-in replacement for /usr/sbin/sendmail to deliver emails sent by unix processes like cron.
+
+If invoked as "sendmail", it will act as sendmail for sending messages. Its
+intention is to let processes like cron send emails. Messages are submitted to
+an actual mail server over SMTP. The destination mail server and credentials are
+configured in /etc/moxsubmit.conf. The From message header is rewritten to the
+configured address.
+
+If submitting an email fails, it is added to a directory moxsubmit.failures in
+the user's home directory.
+
+Most flags are ignored to fake compatibility with other sendmail
+implementations. A single recipient is required, or the tflag.
+
+/etc/moxsubmit.conf should be group-readable and not readable by others and this
+binary should be setgid that group:
+
+	groupadd moxsubmit
+	install -m 2755 -o root -g moxsubmit mox /usr/sbin/sendmail
+	touch /etc/moxsubmit.conf
+	chown root:moxsubmit /etc/moxsubmit.conf
+	chmod 640 /etc/moxsubmit.conf
+	# edit /etc/moxsubmit.conf
+
+
+	usage: mox sendmail [-Fname] [ignoredflags] [-t] [<message]
+
+# mox spf check
+
+Check the status of IP for the policy published in DNS for the domain.
+
+IPs may be allowed to send for a domain, or disallowed, and several shades in
+between. If not allowed, an explanation may be provided by the policy. If so,
+the explanation is printed. The SPF mechanism that matched (if any) is also
+printed.
+
+	usage: mox spf check domain ip
+
+# mox spf lookup
+
+Lookup the SPF record for the domain and print it.
+
+	usage: mox spf lookup domain
+
+# mox spf parse
+
+Parse the record as SPF record. If valid, nothing is printed.
+
+	usage: mox spf parse txtrecord
+
+# mox tlsrpt lookup
+
+Lookup the TLSRPT record for the domain.
+
+A TLSRPT record typically contains an email address where reports about TLS
+connectivity should be sent. Mail servers attempting delivery to our domain
+should attempt to use TLS. TLSRPT lets them report how many connection
+successfully used TLS, and how what kind of errors occurred otherwise.
+
+	usage: mox tlsrpt lookup domain
+
+# mox tlsrpt parsereportmsg
+
+Parse and print the TLSRPT in the message.
+
+The report is printed in formatted JSON.
+
+	usage: mox tlsrpt parsereportmsg message ...
+
+# mox version
+
+Prints this mox version.
+
+	usage: mox version
+*/
+package main
+
+// NOTE: DO NOT EDIT, this file is generated by gendoc.sh.

docker-compose-imaptest.yml

@@ -0,0 +1,31 @@
+version: '3.7'
+services:
+  mox:
+    build: .
+    user: ${MOX_UID}:${MOX_GID}
+    volumes:
+      - ./testdata/imaptest/data:/mox/data
+      - ./testdata/imaptest/mox.conf:/mox/mox.conf
+      - ./testdata/imaptest/domains.conf:/mox/domains.conf
+      - ./testdata/imaptest/imaptest.mbox:/mox/imaptest.mbox
+    working_dir: /mox
+    command: sh -c 'echo testtest | ./mox setaccountpassword mjl@mox.example && ./mox serve'
+    healthcheck:
+      test: netstat -nlt | grep ':1143 '
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+  imaptest:
+    build:
+      dockerfile: Dockerfile.imaptest
+      context: .
+    command: host=mox port=1143 'user=mjl@mox.example' pass=testtest mbox=/imaptest/imaptest.mbox
+    working_dir: /imaptest
+    volumes:
+      - ./testdata/imaptest:/imaptest
+    user: ${MOX_UID}:${MOX_GID}
+    depends_on:
+      mox:
+        condition: service_healthy
+    restart: never

docker-compose-integration.yml

@@ -0,0 +1,95 @@
+version: '3.7'
+services:
+  moxmail:
+    # todo: understand why hostname and/or domainname don't have any influence on the reverse dns set up by docker, requiring us to use our own /etc/resolv.conf...
+    hostname: moxmail1.mox1.example
+    domainname: mox1.example
+    build:
+      dockerfile: Dockerfile.moxmail
+      context: testdata/integration
+    user: ${MOX_UID}:${MOX_GID}
+    volumes:
+      - ./.go:/.go
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
+      - .:/mox
+    environment:
+      GOCACHE: /.go/.cache/go-build
+    command: ["make", "test-postfix"]
+    healthcheck:
+      test: netstat -nlt | grep ':25 '
+      interval: 1s
+      timeout: 1s
+      retries: 10
+    depends_on:
+      dns:
+        condition: service_healthy
+      postfixmail:
+        condition: service_healthy
+    networks:
+      mailnet1:
+        ipv4_address: 172.28.1.10
+      mailnet2:
+        ipv4_address: 172.28.2.10
+      mailnet3:
+        ipv4_address: 172.28.3.10
+
+  postfixmail:
+    hostname: postfixmail.postfix.example
+    domainname: postfix.example
+    build:
+      dockerfile: Dockerfile.postfix
+      context: testdata/integration
+    volumes:
+      # todo: figure out how to mount files with a uid that the process in the container can read...
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
+    command: ["sh", "-c", "set -e; chmod o+r /etc/resolv.conf; (echo 'maillog_file = /dev/stdout'; echo 'mydestination = $$myhostname, localhost.$$mydomain, localhost, $$mydomain') >>/etc/postfix/main.cf; echo 'root: moxtest1@mox1.example' >>/etc/postfix/aliases; newaliases; postfix start-fg"]
+    healthcheck:
+      test: netstat -nlt | grep ':25 '
+      interval: 1s
+      timeout: 1s
+      retries: 10
+    depends_on:
+      dns:
+        condition: service_healthy
+    networks:
+      mailnet1:
+        ipv4_address: 172.28.1.20
+
+  dns:
+    hostname: dns.example
+    build:
+      dockerfile: Dockerfile.dns
+      # todo: figure out how to build from dockerfile with empty context without creating empty dirs in file system.
+      context: testdata/integration
+    volumes:
+      - ./testdata/integration/resolv.conf:/etc/resolv.conf
+      - ./testdata/integration:/integration
+    command: ["sh", "-c", "set -e; chmod o+r /etc/resolv.conf; install -m 640 -o unbound /integration/unbound.conf /integration/*.zone /etc/unbound/; unbound -d -p -v"]
+    healthcheck:
+      test: netstat -nlu | grep '172.28.1.30:53 '
+      interval: 1s
+      timeout: 1s
+      retries: 10
+    networks:
+      mailnet1:
+        ipv4_address: 172.28.1.30
+
+networks:
+  mailnet1:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: "172.28.1.0/24"
+  mailnet2:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: "172.28.2.0/24"
+  mailnet3:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: "172.28.3.0/24"

dsn/dsn.go

@@ -0,0 +1,405 @@
+// Package dsn parses and composes Delivery Status Notification messages, see
+// RFC 3464 and RFC 6533.
+package dsn
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/textproto"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/dkim"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/smtp"
+)
+
+// Message represents a DSN message, with basic message headers, human-readable text,
+// machine-parsable data, and optional original message/headers.
+//
+// A DSN represents a delayed, failed or successful delivery. Failing incoming
+// deliveries over SMTP, and failing outgoing deliveries from the message queue,
+// can result in a DSN being sent.
+type Message struct {
+	SMTPUTF8 bool // Whether the original was received with smtputf8.
+
+	// DSN message From header. E.g. postmaster@ourdomain.example. NOTE:
+	// DSNs should be sent with a null reverse path to prevent mail loops.
+	// ../rfc/3464:421
+	From smtp.Path
+
+	// "To" header, and also SMTP RCP TO to deliver DSN to. Should be taken
+	// from original SMTP transaction MAIL FROM.
+	// ../rfc/3464:415
+	To smtp.Path
+
+	// Message subject header, e.g. describing mail delivery failure.
+	Subject string
+
+	// Human-readable text explaining the failure. Line endings should be
+	// bare newlines, not \r\n. They are converted to \r\n when composing.
+	TextBody string
+
+	// Per-message fields.
+	OriginalEnvelopeID string
+	ReportingMTA       string // Required.
+	DSNGateway         string
+	ReceivedFromMTA    smtp.Ehlo // Host from which message was received.
+	ArrivalDate        time.Time
+
+	// All per-message fields, including extensions. Only used for parsing,
+	// not composing.
+	MessageHeader textproto.MIMEHeader
+
+	// One or more per-recipient fields.
+	// ../rfc/3464:436
+	Recipients []Recipient
+
+	// Original message or headers to include in DSN as third MIME part.
+	// Optional. Only used for generating DSNs, not set for parsed DNSs.
+	Original []byte
+}
+
+// Action is a field in a DSN.
+type Action string
+
+// ../rfc/3464:890
+
+const (
+	Failed    Action = "failed"
+	Delayed   Action = "delayed"
+	Delivered Action = "delivered"
+	Relayed   Action = "relayed"
+	Expanded  Action = "expanded"
+)
+
+// ../rfc/3464:1530 ../rfc/6533:370
+
+// Recipient holds the per-recipient delivery-status lines in a DSN.
+type Recipient struct {
+	// Required fields.
+	FinalRecipient smtp.Path // Final recipient of message.
+	Action         Action
+
+	// Enhanced status code. First digit indicates permanent or temporary
+	// error. If the string contains more than just a status, that
+	// additional text is added as comment when composing a DSN.
+	Status string
+
+	// Optional fields.
+	// Original intended recipient of message. Used with the DSN extensions ORCPT
+	// parameter.
+	// ../rfc/3464:1197
+	OriginalRecipient smtp.Path
+
+	// Remote host that returned an error code. Can also be empty for
+	// deliveries.
+	RemoteMTA NameIP
+
+	// If RemoteMTA is present, DiagnosticCode is from remote. When
+	// creating a DSN, additional text in the string will be added to the
+	// DSN as comment.
+	DiagnosticCode  string
+	LastAttemptDate time.Time
+	FinalLogID      string
+
+	// For delayed deliveries, deliveries may be retried until this time.
+	WillRetryUntil *time.Time
+
+	// All fields, including extensions. Only used for parsing, not
+	// composing.
+	Header textproto.MIMEHeader
+}
+
+// Compose returns a DSN message.
+//
+// smtputf8 indicates whether the remote MTA that is receiving the DSN
+// supports smtputf8. This influences the message media (sub)types used for the
+// DSN.
+//
+// DKIM signatures are added if DKIM signing is configured for the "from" domain.
+func (m *Message) Compose(log *mlog.Log, smtputf8 bool) ([]byte, error) {
+	// ../rfc/3462:119
+	// ../rfc/3464:377
+	// We'll make a multipart/report with 2 or 3 parts:
+	// - 1. human-readable explanation;
+	// - 2. message/delivery-status;
+	// - 3. (optional) original message (either in full, or only headers).
+
+	// todo future: add option to send full message. but only do so if the message is <100kb.
+	// todo future: possibly write to a file directly, instead of building up message in memory.
+
+	// If message does not require smtputf8, we are never generating a utf-8 DSN.
+	if !m.SMTPUTF8 {
+		smtputf8 = false
+	}
+
+	// We check for errors once after all the writes.
+	msgw := &errWriter{w: &bytes.Buffer{}}
+
+	header := func(k, v string) {
+		fmt.Fprintf(msgw, "%s: %s\r\n", k, v)
+	}
+
+	line := func(w io.Writer) {
+		w.Write([]byte("\r\n"))
+	}
+
+	// Outer message headers.
+	header("From", fmt.Sprintf("<%s>", m.From.XString(smtputf8))) // todo: would be good to have a local ascii-only name for this address.
+	header("To", fmt.Sprintf("<%s>", m.To.XString(smtputf8)))     // todo: we could just leave this out if it has utf-8 and remote does not support utf-8.
+	header("Subject", m.Subject)
+	header("Message-Id", fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8)))
+	header("Date", time.Now().Format(message.RFC5322Z))
+	header("MIME-Version", "1.0")
+	mp := multipart.NewWriter(msgw)
+	header("Content-Type", fmt.Sprintf(`multipart/report; report-type="delivery-status"; boundary="%s"`, mp.Boundary()))
+
+	line(msgw)
+
+	// First part, human-readable message.
+	msgHdr := textproto.MIMEHeader{}
+	if smtputf8 {
+		msgHdr.Set("Content-Type", "text/plain; charset=utf-8")
+		msgHdr.Set("Content-Transfer-Encoding", "8BIT")
+	} else {
+		msgHdr.Set("Content-Type", "text/plain")
+		msgHdr.Set("Content-Transfer-Encoding", "7BIT")
+	}
+	msgp, err := mp.CreatePart(msgHdr)
+	if err != nil {
+		return nil, err
+	}
+	msgp.Write([]byte(strings.ReplaceAll(m.TextBody, "\n", "\r\n")))
+
+	// Machine-parsable message. ../rfc/3464:455
+	statusHdr := textproto.MIMEHeader{}
+	if smtputf8 {
+		// ../rfc/6533:325
+		statusHdr.Set("Content-Type", "message/global-delivery-status")
+		statusHdr.Set("Content-Transfer-Encoding", "8BIT")
+	} else {
+		statusHdr.Set("Content-Type", "message/delivery-status")
+		statusHdr.Set("Content-Transfer-Encoding", "7BIT")
+	}
+	statusp, err := mp.CreatePart(statusHdr)
+	if err != nil {
+		return nil, err
+	}
+
+	// ../rfc/3464:470
+	// examples: ../rfc/3464:1855
+	// type fields: ../rfc/3464:536 https://www.iana.org/assignments/dsn-types/dsn-types.xhtml
+
+	status := func(k, v string) {
+		fmt.Fprintf(statusp, "%s: %s\r\n", k, v)
+	}
+
+	// Per-message fields first. ../rfc/3464:575
+	// todo future: once we support the smtp dsn extension, the envid should be saved/set as OriginalEnvelopeID. ../rfc/3464:583 ../rfc/3461:1139
+	if m.OriginalEnvelopeID != "" {
+		status("Original-Envelope-ID", m.OriginalEnvelopeID)
+	}
+	status("Reporting-MTA", "dns; "+m.ReportingMTA) // ../rfc/3464:628
+	if m.DSNGateway != "" {
+		// ../rfc/3464:714
+		status("DSN-Gateway", "dns; "+m.DSNGateway)
+	}
+	if !m.ReceivedFromMTA.IsZero() {
+		// ../rfc/3464:735
+		status("Received-From-MTA", fmt.Sprintf("dns;%s (%s)", m.ReceivedFromMTA.Name, smtp.AddressLiteral(m.ReceivedFromMTA.ConnIP)))
+	}
+	status("Arrival-Date", m.ArrivalDate.Format(message.RFC5322Z)) // ../rfc/3464:758
+
+	// Then per-recipient fields. ../rfc/3464:769
+	// todo: should also handle other address types. at least recognize "unknown". Probably just store this field. ../rfc/3464:819
+	addrType := "rfc822;" // ../rfc/3464:514
+	if smtputf8 {
+		addrType = "utf-8;" // ../rfc/6533:250
+	}
+	if len(m.Recipients) == 0 {
+		return nil, fmt.Errorf("missing per-recipient fields")
+	}
+	for _, r := range m.Recipients {
+		line(statusp)
+		if !r.OriginalRecipient.IsZero() {
+			// ../rfc/3464:807
+			status("Original-Recipient", addrType+r.OriginalRecipient.DSNString(smtputf8))
+		}
+		status("Final-Recipient", addrType+r.FinalRecipient.DSNString(smtputf8)) // ../rfc/3464:829
+		status("Action", string(r.Action))                                       // ../rfc/3464:879
+		st := r.Status
+		if st == "" {
+			// ../rfc/3464:944
+			// Making up a status code is not great, but the field is required. We could simply
+			// require the caller to make one up...
+			switch r.Action {
+			case Delayed:
+				st = "4.0.0"
+			case Failed:
+				st = "5.0.0"
+			default:
+				st = "2.0.0"
+			}
+		}
+		var rest string
+		st, rest = codeLine(st)
+		statusLine := st
+		if rest != "" {
+			statusLine += " (" + rest + ")"
+		}
+		status("Status", statusLine) // ../rfc/3464:975
+		if !r.RemoteMTA.IsZero() {
+			// ../rfc/3464:1015
+			status("Remote-MTA", fmt.Sprintf("dns;%s (%s)", r.RemoteMTA.Name, smtp.AddressLiteral(r.RemoteMTA.IP)))
+		}
+		// Presence of Diagnostic-Code indicates the code is from Remote-MTA. ../rfc/3464:1053
+		if r.DiagnosticCode != "" {
+			diagCode, rest := codeLine(r.DiagnosticCode)
+			diagLine := diagCode
+			if rest != "" {
+				diagLine += " (" + rest + ")"
+			}
+			// ../rfc/6533:589
+			status("Diagnostic-Code", "smtp; "+diagLine)
+		}
+		if !r.LastAttemptDate.IsZero() {
+			status("Last-Attempt-Date", r.LastAttemptDate.Format(message.RFC5322Z)) // ../rfc/3464:1076
+		}
+		if r.FinalLogID != "" {
+			// todo future: think about adding cid as "Final-Log-Id"?
+			status("Final-Log-ID", r.FinalLogID) // ../rfc/3464:1098
+		}
+		if r.WillRetryUntil != nil {
+			status("Will-Retry-Until", r.WillRetryUntil.Format(message.RFC5322Z)) // ../rfc/3464:1108
+		}
+	}
+
+	// We include only the header of the original message.
+	// todo: add the textual version of the original message, if it exists and isn't too large.
+	if m.Original != nil {
+		headers, err := message.ReadHeaders(bufio.NewReader(bytes.NewReader(m.Original)))
+		if err != nil && errors.Is(err, message.ErrHeaderSeparator) {
+			// Whole data is a header.
+			headers = m.Original
+		} else if err != nil {
+			return nil, err
+		} else {
+			// This is a whole message. We still only include the headers.
+			// todo: include the whole body.
+		}
+
+		origHdr := textproto.MIMEHeader{}
+		if smtputf8 {
+			// ../rfc/6533:431
+			// ../rfc/6533:605
+			origHdr.Set("Content-Type", "message/global-headers") // ../rfc/6533:625
+			origHdr.Set("Content-Transfer-Encoding", "8BIT")
+		} else {
+			// ../rfc/3462:175
+			if m.SMTPUTF8 {
+				// ../rfc/6533:480
+				origHdr.Set("Content-Type", "text/rfc822-headers; charset=utf-8")
+				origHdr.Set("Content-Transfer-Encoding", "BASE64")
+			} else {
+				origHdr.Set("Content-Type", "text/rfc822-headers")
+				origHdr.Set("Content-Transfer-Encoding", "7BIT")
+			}
+		}
+		origp, err := mp.CreatePart(origHdr)
+		if err != nil {
+			return nil, err
+		}
+
+		if !smtputf8 && m.SMTPUTF8 {
+			data := base64.StdEncoding.EncodeToString(headers)
+			for len(data) > 0 {
+				line := data
+				n := len(line)
+				if n > 78 {
+					n = 78
+				}
+				line, data = data[:n], data[n:]
+				origp.Write([]byte(line + "\r\n"))
+			}
+		} else {
+			origp.Write(headers)
+		}
+	}
+
+	if err := mp.Close(); err != nil {
+		return nil, err
+	}
+
+	if msgw.err != nil {
+		return nil, err
+	}
+
+	data := msgw.w.Bytes()
+
+	fd := m.From.IPDomain.Domain
+	confDom, _ := mox.Conf.Domain(fd)
+	if len(confDom.DKIM.Sign) > 0 {
+		if dkimHeaders, err := dkim.Sign(context.Background(), m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data)); err != nil {
+			log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, mlog.Field("domain", fd))
+		} else {
+			data = append([]byte(dkimHeaders), data...)
+		}
+	}
+
+	return data, nil
+}
+
+type errWriter struct {
+	w   *bytes.Buffer
+	err error
+}
+
+func (w *errWriter) Write(buf []byte) (int, error) {
+	if w.err != nil {
+		return -1, w.err
+	}
+	n, err := w.w.Write(buf)
+	w.err = err
+	return n, err
+}
+
+// split a line into enhanced status code and rest.
+func codeLine(s string) (string, string) {
+	t := strings.SplitN(s, " ", 2)
+	l := strings.Split(t[0], ".")
+	if len(l) != 3 {
+		return "", s
+	}
+	for i, e := range l {
+		_, err := strconv.ParseInt(e, 10, 32)
+		if err != nil {
+			return "", s
+		}
+		if i == 0 && len(e) != 1 {
+			return "", s
+		}
+	}
+
+	var rest string
+	if len(t) == 2 {
+		rest = t[1]
+	}
+	return t[0], rest
+}
+
+// HasCode returns whether line starts with an enhanced SMTP status code.
+func HasCode(line string) bool {
+	// ../rfc/3464:986
+	ecode, _ := codeLine(line)
+	return ecode != ""
+}

dsn/dsn_test.go

@@ -0,0 +1,243 @@
+package dsn
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/dkim"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/smtp"
+)
+
+func xparseDomain(s string) dns.Domain {
+	d, err := dns.ParseDomain(s)
+	if err != nil {
+		panic(fmt.Sprintf("parsing domain %q: %v", s, err))
+	}
+	return d
+}
+
+func xparseIPDomain(s string) dns.IPDomain {
+	return dns.IPDomain{Domain: xparseDomain(s)}
+}
+
+func tparseMessage(t *testing.T, data []byte, nparts int) (*Message, *message.Part) {
+	t.Helper()
+	m, p, err := Parse(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("parsing dsn: %v", err)
+	}
+	if len(p.Parts) != nparts {
+		t.Fatalf("got %d parts, expected %d", len(p.Parts), nparts)
+	}
+	return m, p
+}
+
+func tcheckType(t *testing.T, p *message.Part, mt, mst, cte string) {
+	t.Helper()
+	if !strings.EqualFold(p.MediaType, mt) {
+		t.Fatalf("got mediatype %q, expected %q", p.MediaType, mt)
+	}
+	if !strings.EqualFold(p.MediaSubType, mst) {
+		t.Fatalf("got mediasubtype %q, expected %q", p.MediaSubType, mst)
+	}
+	if !strings.EqualFold(p.ContentTransferEncoding, cte) {
+		t.Fatalf("got content-transfer-encoding %q, expected %q", p.ContentTransferEncoding, cte)
+	}
+}
+
+func tcompare(t *testing.T, got, exp any) {
+	t.Helper()
+	if !reflect.DeepEqual(got, exp) {
+		t.Fatalf("got %#v, expected %#v", got, exp)
+	}
+}
+
+func tcompareReader(t *testing.T, r io.Reader, exp []byte) {
+	t.Helper()
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		t.Fatalf("data read, got %q, expected %q", buf, exp)
+	}
+}
+
+func TestDSN(t *testing.T) {
+	log := mlog.New("dsn")
+
+	now := time.Now()
+
+	// An ascii-only message.
+	m := Message{
+		SMTPUTF8: false,
+
+		From:     smtp.Path{Localpart: "postmaster", IPDomain: xparseIPDomain("mox.example")},
+		To:       smtp.Path{Localpart: "mjl", IPDomain: xparseIPDomain("remote.example")},
+		Subject:  "dsn",
+		TextBody: "delivery failure\n",
+
+		ReportingMTA:    "mox.example",
+		ReceivedFromMTA: smtp.Ehlo{Name: xparseIPDomain("relay.example"), ConnIP: net.ParseIP("10.10.10.10")},
+		ArrivalDate:     now,
+
+		Recipients: []Recipient{
+			{
+				FinalRecipient:  smtp.Path{Localpart: "mjl", IPDomain: xparseIPDomain("remote.example")},
+				Action:          Failed,
+				Status:          "5.0.0",
+				LastAttemptDate: now,
+			},
+		},
+
+		Original: []byte("Subject: test\r\n"),
+	}
+	msgbuf, err := m.Compose(log, false)
+	if err != nil {
+		t.Fatalf("composing dsn: %v", err)
+	}
+	pmsg, part := tparseMessage(t, msgbuf, 3)
+	tcheckType(t, part, "multipart", "report", "")
+	tcheckType(t, &part.Parts[0], "text", "plain", "7bit")
+	tcheckType(t, &part.Parts[1], "message", "delivery-status", "7bit")
+	tcheckType(t, &part.Parts[2], "text", "rfc822-headers", "7bit")
+	tcompare(t, part.Parts[2].ContentTypeParams["charset"], "")
+	tcompareReader(t, part.Parts[2].Reader(), m.Original)
+	tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient)
+	// todo: test more fields
+
+	msgbufutf8, err := m.Compose(log, true)
+	if err != nil {
+		t.Fatalf("composing dsn with utf-8: %v", err)
+	}
+	pmsg, part = tparseMessage(t, msgbufutf8, 3)
+	tcheckType(t, part, "multipart", "report", "")
+	tcheckType(t, &part.Parts[0], "text", "plain", "7bit")
+	tcheckType(t, &part.Parts[1], "message", "delivery-status", "7bit")
+	tcheckType(t, &part.Parts[2], "text", "rfc822-headers", "7bit")
+	tcompare(t, part.Parts[2].ContentTypeParams["charset"], "")
+	tcompareReader(t, part.Parts[2].Reader(), m.Original)
+	tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient)
+
+	// Test for valid DKIM signature.
+	mox.Context = context.Background()
+	mox.ConfigStaticPath = "../testdata/dsn/mox.conf"
+	mox.MustLoadConfig()
+	msgbuf, err = m.Compose(log, false)
+	if err != nil {
+		t.Fatalf("composing utf-8 dsn with utf-8 support: %v", err)
+	}
+	resolver := &dns.MockResolver{
+		TXT: map[string][]string{
+			"testsel._domainkey.mox.example.": {"v=DKIM1;h=sha256;t=s;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3ZId3ys70VFspp/VMFaxMOrNjHNPg04NOE1iShih16b3Ex7hHBOgC1UvTGSmrMlbCB1OxTXkvf6jW6S4oYRnZYVNygH6zKUwYYhaSaGIg1xA/fDn+IgcTRyLoXizMUgUgpTGyxhNrwIIWv+i7jjbs3TKpP3NU4owQ/rxowmSNqg+fHIF1likSvXvljYS" + "jaFXXnWfYibW7TdDCFFpN4sB5o13+as0u4vLw6MvOi59B1tLype1LcHpi1b9PfxNtznTTdet3kL0paxIcWtKHT0LDPUos8YYmiPa5nGbUqlC7d+4YT2jQPvwGxCws1oo2Tw6nj1UaihneYGAyvEky49FBwIDAQAB"},
+		},
+	}
+	results, err := dkim.Verify(context.Background(), resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false)
+	if err != nil {
+		t.Fatalf("dkim verify: %v", err)
+	}
+	if len(results) != 1 || results[0].Status != dkim.StatusPass {
+		t.Fatalf("dkim result not pass, %#v", results)
+	}
+
+	// An utf-8 message.
+	m = Message{
+		SMTPUTF8: true,
+
+		From:     smtp.Path{Localpart: "postmæster", IPDomain: xparseIPDomain("møx.example")},
+		To:       smtp.Path{Localpart: "møx", IPDomain: xparseIPDomain("remøte.example")},
+		Subject:  "dsn¡",
+		TextBody: "delivery failure¿\n",
+
+		ReportingMTA:    "mox.example",
+		ReceivedFromMTA: smtp.Ehlo{Name: xparseIPDomain("reläy.example"), ConnIP: net.ParseIP("10.10.10.10")},
+		ArrivalDate:     now,
+
+		Recipients: []Recipient{
+			{
+				Action:          Failed,
+				FinalRecipient:  smtp.Path{Localpart: "møx", IPDomain: xparseIPDomain("remøte.example")},
+				Status:          "5.0.0",
+				LastAttemptDate: now,
+			},
+		},
+
+		Original: []byte("Subject: tést\r\n"),
+	}
+	msgbuf, err = m.Compose(log, false)
+	if err != nil {
+		t.Fatalf("composing utf-8 dsn without utf-8 support: %v", err)
+	}
+	pmsg, part = tparseMessage(t, msgbuf, 3)
+	tcheckType(t, part, "multipart", "report", "")
+	tcheckType(t, &part.Parts[0], "text", "plain", "7bit")
+	tcheckType(t, &part.Parts[1], "message", "delivery-status", "7bit")
+	tcheckType(t, &part.Parts[2], "text", "rfc822-headers", "base64")
+	tcompare(t, part.Parts[2].ContentTypeParams["charset"], "utf-8")
+	tcompareReader(t, part.Parts[2].Reader(), m.Original)
+	tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient)
+
+	msgbufutf8, err = m.Compose(log, true)
+	if err != nil {
+		t.Fatalf("composing utf-8 dsn with utf-8 support: %v", err)
+	}
+	pmsg, part = tparseMessage(t, msgbufutf8, 3)
+	tcheckType(t, part, "multipart", "report", "")
+	tcheckType(t, &part.Parts[0], "text", "plain", "8bit")
+	tcheckType(t, &part.Parts[1], "message", "global-delivery-status", "8bit")
+	tcheckType(t, &part.Parts[2], "message", "global-headers", "8bit")
+	tcompare(t, part.Parts[2].ContentTypeParams["charset"], "")
+	tcompareReader(t, part.Parts[2].Reader(), m.Original)
+	tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient)
+
+	// Now a message without 3rd multipart.
+	m.Original = nil
+	msgbufutf8, err = m.Compose(log, true)
+	if err != nil {
+		t.Fatalf("composing utf-8 dsn with utf-8 support: %v", err)
+	}
+	pmsg, part = tparseMessage(t, msgbufutf8, 2)
+	tcheckType(t, part, "multipart", "report", "")
+	tcheckType(t, &part.Parts[0], "text", "plain", "8bit")
+	tcheckType(t, &part.Parts[1], "message", "global-delivery-status", "8bit")
+	tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient)
+}
+
+func TestCode(t *testing.T) {
+	testCodeLine := func(line, ecode, rest string) {
+		t.Helper()
+		e, r := codeLine(line)
+		if e != ecode || r != rest {
+			t.Fatalf("codeLine %q: got %q %q, expected %q %q", line, e, r, ecode, rest)
+		}
+	}
+	testCodeLine("4.0.0", "4.0.0", "")
+	testCodeLine("4.0.0 more", "4.0.0", "more")
+	testCodeLine("other", "", "other")
+	testCodeLine("other more", "", "other more")
+
+	testHasCode := func(line string, exp bool) {
+		t.Helper()
+		got := HasCode(line)
+		if got != exp {
+			t.Fatalf("HasCode %q: got %v, expected %v", line, got, exp)
+		}
+	}
+	testHasCode("4.0.0", true)
+	testHasCode("5.7.28", true)
+	testHasCode("10.0.0", false) // first number must be single digit.
+	testHasCode("4.1.1 more", true)
+	testHasCode("other ", false)
+	testHasCode("4.2.", false)
+	testHasCode("4.2. ", false)
+	testHasCode(" 4.2.4", false)
+	testHasCode(" 4.2.4 ", false)
+}

dsn/nameip.go

@@ -0,0 +1,15 @@
+package dsn
+
+import (
+	"net"
+)
+
+// NameIP represents a name and possibly IP, e.g. representing a connection destination.
+type NameIP struct {
+	Name string
+	IP   net.IP
+}
+
+func (n NameIP) IsZero() bool {
+	return n.Name == "" && n.IP == nil
+}

dsn/parse.go

@@ -0,0 +1,360 @@
+package dsn
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/textproto"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/smtp"
+)
+
+// Parse reads a DSN message.
+//
+// A DSN is a multipart internet mail message with 2 or 3 parts: human-readable
+// text, machine-parsable text, and optional original message or headers.
+//
+// The first return value is the machine-parsed DSN message. The second value is
+// the entire MIME multipart message. Use its Parts field to access the
+// human-readable text and optional original message/headers.
+func Parse(r io.ReaderAt) (*Message, *message.Part, error) {
+	// DSNs can mix and match subtypes with and without utf-8. ../rfc/6533:441
+
+	part, err := message.Parse(r)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parsing message: %v", err)
+	}
+	if part.MediaType != "MULTIPART" || part.MediaSubType != "REPORT" {
+		return nil, nil, fmt.Errorf(`message has content-type %q, must have "message/report"`, strings.ToLower(part.MediaType+"/"+part.MediaSubType))
+	}
+	err = part.Walk()
+	if err != nil {
+		return nil, nil, fmt.Errorf("parsing message parts: %v", err)
+	}
+	nparts := len(part.Parts)
+	if nparts != 2 && nparts != 3 {
+		return nil, nil, fmt.Errorf("invalid dsn, got %d multipart parts, 2 or 3 required", nparts)
+	}
+	p0 := part.Parts[0]
+	if !(p0.MediaType == "" && p0.MediaSubType == "") && !(p0.MediaType == "TEXT" && p0.MediaSubType == "PLAIN") {
+		return nil, nil, fmt.Errorf(`invalid dsn, first part has content-type %q, must have "text/plain"`, strings.ToLower(p0.MediaType+"/"+p0.MediaSubType))
+	}
+
+	p1 := part.Parts[1]
+	var m *Message
+	if !(p1.MediaType == "MESSAGE" && (p1.MediaSubType == "DELIVERY-STATUS" || p1.MediaSubType == "GLOBAL-DELIVERY-STATUS")) {
+		return nil, nil, fmt.Errorf(`invalid dsn, second part has content-type %q, must have "message/delivery-status" or "message/global-delivery-status"`, strings.ToLower(p1.MediaType+"/"+p1.MediaSubType))
+	}
+	utf8 := p1.MediaSubType == "GLOBAL-DELIVERY-STATUS"
+	m, err = Decode(p1.Reader(), utf8)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parsing dsn delivery-status part: %v", err)
+	}
+
+	addressPath := func(a message.Address) (smtp.Path, error) {
+		d, err := dns.ParseDomain(a.Host)
+		if err != nil {
+			return smtp.Path{}, fmt.Errorf("parsing domain: %v", err)
+		}
+		return smtp.Path{Localpart: smtp.Localpart(a.User), IPDomain: dns.IPDomain{Domain: d}}, nil
+	}
+	if len(part.Envelope.From) == 1 {
+		m.From, err = addressPath(part.Envelope.From[0])
+		if err != nil {
+			return nil, nil, fmt.Errorf("parsing From-header: %v", err)
+		}
+	}
+	if len(part.Envelope.To) == 1 {
+		m.To, err = addressPath(part.Envelope.To[0])
+		if err != nil {
+			return nil, nil, fmt.Errorf("parsing To-header: %v", err)
+		}
+	}
+	m.Subject = part.Envelope.Subject
+	buf, err := io.ReadAll(p0.Reader())
+	if err != nil {
+		return nil, nil, fmt.Errorf("reading human-readable text part: %v", err)
+	}
+	m.TextBody = strings.ReplaceAll(string(buf), "\r\n", "\n")
+
+	if nparts == 2 {
+		return m, &part, nil
+	}
+
+	p2 := part.Parts[2]
+	ct := strings.ToLower(p2.MediaType + "/" + p2.MediaSubType)
+	switch ct {
+	case "text/rfc822-headers":
+	case "message/global-headers":
+	case "message/rfc822":
+	case "message/global":
+	default:
+		return nil, nil, fmt.Errorf("invalid content-type %q for optional third part with original message/headers", ct)
+	}
+
+	return m, &part, nil
+}
+
+// Decode parses the (global) delivery-status part of a DSN.
+//
+// utf8 indicates if UTF-8 is allowed for this message, if used by the media
+// subtype of the message parts.
+func Decode(r io.Reader, utf8 bool) (*Message, error) {
+	m := Message{SMTPUTF8: utf8}
+
+	// We are using textproto.Reader to read mime headers. It requires a header section ending in \r\n.
+	// ../rfc/3464:486
+	b := bufio.NewReader(io.MultiReader(r, strings.NewReader("\r\n")))
+	mr := textproto.NewReader(b)
+
+	// Read per-message lines.
+	// ../rfc/3464:1522 ../rfc/6533:366
+	msgh, err := mr.ReadMIMEHeader()
+	if err != nil {
+		return nil, fmt.Errorf("reading per-message lines: %v", err)
+	}
+	for k, l := range msgh {
+		if len(l) != 1 {
+			return nil, fmt.Errorf("multiple values for %q: %v", k, l)
+		}
+		v := l[0]
+		// note: headers are in canonical form, as parsed by textproto.
+		switch k {
+		case "Original-Envelope-Id":
+			m.OriginalEnvelopeID = v
+		case "Reporting-Mta":
+			mta, err := parseMTA(v, utf8)
+			if err != nil {
+				return nil, fmt.Errorf("parsing reporting-mta: %v", err)
+			}
+			m.ReportingMTA = mta
+		case "Dsn-Gateway":
+			mta, err := parseMTA(v, utf8)
+			if err != nil {
+				return nil, fmt.Errorf("parsing dsn-gateway: %v", err)
+			}
+			m.DSNGateway = mta
+		case "Received-From-Mta":
+			mta, err := parseMTA(v, utf8)
+			if err != nil {
+				return nil, fmt.Errorf("parsing received-from-mta: %v", err)
+			}
+			d, err := dns.ParseDomain(mta)
+			if err != nil {
+				return nil, fmt.Errorf("parsing received-from-mta domain %q: %v", mta, err)
+			}
+			m.ReceivedFromMTA = smtp.Ehlo{Name: dns.IPDomain{Domain: d}}
+		case "Arrival-Date":
+			tm, err := parseDateTime(v)
+			if err != nil {
+				return nil, fmt.Errorf("parsing arrival-date: %v", err)
+			}
+			m.ArrivalDate = tm
+		default:
+			// We'll assume it is an extension field, we'll ignore it for now.
+		}
+	}
+	m.MessageHeader = msgh
+
+	required := []string{"Reporting-Mta"}
+	for _, req := range required {
+		if _, ok := msgh[req]; !ok {
+			return nil, fmt.Errorf("missing required recipient field %q", req)
+		}
+	}
+
+	rh, err := parseRecipientHeader(mr, utf8)
+	if err != nil {
+		return nil, fmt.Errorf("reading per-recipient header: %v", err)
+	}
+	m.Recipients = []Recipient{rh}
+	for {
+		if _, err := b.Peek(1); err == io.EOF {
+			break
+		}
+		rh, err := parseRecipientHeader(mr, utf8)
+		if err != nil {
+			return nil, fmt.Errorf("reading another per-recipient header: %v", err)
+		}
+		m.Recipients = append(m.Recipients, rh)
+	}
+	return &m, nil
+}
+
+// ../rfc/3464:1530 ../rfc/6533:370
+func parseRecipientHeader(mr *textproto.Reader, utf8 bool) (Recipient, error) {
+	var r Recipient
+	h, err := mr.ReadMIMEHeader()
+	if err != nil {
+		return Recipient{}, err
+	}
+
+	for k, l := range h {
+		if len(l) != 1 {
+			return Recipient{}, fmt.Errorf("multiple values for %q: %v", k, l)
+		}
+		v := l[0]
+		// note: headers are in canonical form, as parsed by textproto.
+		var err error
+		switch k {
+		case "Original-Recipient":
+			r.OriginalRecipient, err = parseAddress(v, utf8)
+		case "Final-Recipient":
+			r.FinalRecipient, err = parseAddress(v, utf8)
+		case "Action":
+			a := Action(strings.ToLower(v))
+			actions := []Action{Failed, Delayed, Delivered, Relayed, Expanded}
+			var ok bool
+			for _, x := range actions {
+				if a == x {
+					ok = true
+					break
+				}
+			}
+			if !ok {
+				err = fmt.Errorf("unrecognized action %q", v)
+			}
+		case "Status":
+			// todo: parse the enhanced status code?
+			r.Status = v
+		case "Remote-Mta":
+			r.RemoteMTA = NameIP{Name: v}
+		case "Diagnostic-Code":
+			// ../rfc/3464:518
+			t := strings.SplitN(v, ";", 2)
+			dt := strings.TrimSpace(t[0])
+			if strings.ToLower(dt) != "smtp" {
+				err = fmt.Errorf("unknown diagnostic-type %q, expected smtp", dt)
+			} else if len(t) != 2 {
+				err = fmt.Errorf("missing semicolon to separate diagnostic-type from code")
+			} else {
+				r.DiagnosticCode = strings.TrimSpace(t[1])
+			}
+		case "Last-Attempt-Date":
+			r.LastAttemptDate, err = parseDateTime(v)
+		case "Final-Log-Id":
+			r.FinalLogID = v
+		case "Will-Retry-Until":
+			tm, err := parseDateTime(v)
+			if err == nil {
+				r.WillRetryUntil = &tm
+			}
+		default:
+			// todo future: parse localized diagnostic text field?
+			// We'll assume it is an extension field, we'll ignore it for now.
+		}
+		if err != nil {
+			return Recipient{}, fmt.Errorf("parsing field %q %q: %v", k, v, err)
+		}
+	}
+
+	required := []string{"Final-Recipient", "Action", "Status"}
+	for _, req := range required {
+		if _, ok := h[req]; !ok {
+			return Recipient{}, fmt.Errorf("missing required recipient field %q", req)
+		}
+	}
+
+	r.Header = h
+	return r, nil
+}
+
+// ../rfc/3464:525
+func parseMTA(s string, utf8 bool) (string, error) {
+	s = removeComments(s)
+	t := strings.SplitN(s, ";", 2)
+	if len(t) != 2 {
+		return "", fmt.Errorf("missing semicolon that splits type and name")
+	}
+	k := strings.TrimSpace(t[0])
+	if !strings.EqualFold(k, "dns") {
+		return "", fmt.Errorf("unknown type %q, expected dns", k)
+	}
+	return strings.TrimSpace(t[1]), nil
+}
+
+func parseDateTime(s string) (time.Time, error) {
+	s = removeComments(s)
+	return time.Parse(message.RFC5322Z, s)
+}
+
+func parseAddress(s string, utf8 bool) (smtp.Path, error) {
+	s = removeComments(s)
+	t := strings.SplitN(s, ";", 2)
+	// ../rfc/3464:513 ../rfc/6533:250
+	addrType := strings.ToLower(strings.TrimSpace(t[0]))
+	if len(t) != 2 {
+		return smtp.Path{}, fmt.Errorf("missing semicolon that splits address type and address")
+	} else if addrType == "utf-8" {
+		if !utf8 {
+			return smtp.Path{}, fmt.Errorf("utf-8 address type for non-utf-8 dsn")
+		}
+	} else if addrType != "rfc822" {
+		return smtp.Path{}, fmt.Errorf("unrecognized address type %q, expected rfc822", addrType)
+	}
+	s = strings.TrimSpace(t[1])
+	if !utf8 {
+		for _, c := range s {
+			if c > 0x7f {
+				return smtp.Path{}, fmt.Errorf("non-ascii without utf-8 enabled")
+			}
+		}
+	}
+	// todo: more proper parser
+	t = strings.SplitN(s, "@", 2)
+	if len(t) != 2 || t[0] == "" || t[1] == "" {
+		return smtp.Path{}, fmt.Errorf("invalid email address")
+	}
+	d, err := dns.ParseDomain(t[1])
+	if err != nil {
+		return smtp.Path{}, fmt.Errorf("parsing domain: %v", err)
+	}
+	var lp string
+	var esc string
+	for _, c := range t[0] {
+		if esc == "" && c == '\\' || esc == `\` && (c == 'x' || c == 'X') || esc == `\x` && c == '{' {
+			if c == 'X' {
+				c = 'x'
+			}
+			esc += string(c)
+		} else if strings.HasPrefix(esc, `\x{`) {
+			if c == '}' {
+				c, err := strconv.ParseInt(esc[3:], 16, 32)
+				if err != nil {
+					return smtp.Path{}, fmt.Errorf("parsing localpart with hexpoint: %v", err)
+				}
+				lp += string(rune(c))
+				esc = ""
+			} else {
+				esc += string(c)
+			}
+		} else {
+			lp += string(c)
+		}
+	}
+	if esc != "" {
+		return smtp.Path{}, fmt.Errorf("parsing localpart: unfinished embedded unicode char")
+	}
+	p := smtp.Path{Localpart: smtp.Localpart(lp), IPDomain: dns.IPDomain{Domain: d}}
+	return p, nil
+}
+
+func removeComments(s string) string {
+	n := 0
+	r := ""
+	for _, c := range s {
+		if c == '(' {
+			n++
+		} else if c == ')' && n > 0 {
+			n--
+		} else if n == 0 {
+			r += string(c)
+		}
+	}
+	return r
+}

export.go

@@ -0,0 +1,264 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/store"
+)
+
+func cmdExportMaildir(c *cmd) {
+	c.params = "dst-path account-path [mailbox]"
+	c.help = `Export one or all mailboxes from an account in maildir format.
+
+Export bypasses a running mox instance. It opens the account mailbox/message
+database file directly. This may block if a running mox instance also has the
+database open, e.g. for IMAP connections.
+`
+	args := c.Parse()
+	xcmdExport(false, args, c)
+}
+
+func cmdExportMbox(c *cmd) {
+	c.params = "dst-path account-path [mailbox]"
+	c.help = `Export messages from one or all mailboxes in an account in mbox format.
+
+Using mbox is not recommended. Maildir is a better format.
+
+Export bypasses a running mox instance. It opens the account mailbox/message
+database file directly. This may block if a running mox instance also has the
+database open, e.g. for IMAP connections.
+
+For mbox export, we use "mboxrd" where message lines starting with the magic
+"From " string are escaped by prepending a >. We escape all ">*From ",
+otherwise reconstructing the original could lose a ">".
+`
+	args := c.Parse()
+	xcmdExport(true, args, c)
+}
+
+func xcmdExport(mbox bool, args []string, c *cmd) {
+	if len(args) != 2 && len(args) != 3 {
+		c.Usage()
+	}
+
+	dst := args[0]
+	accountDir := args[1]
+	var mailbox string
+	if len(args) == 3 {
+		mailbox = args[2]
+	}
+
+	dbpath := filepath.Join(accountDir, "index.db")
+	db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{})
+	xcheckf(err, "open database %q", dbpath)
+
+	err = db.Read(func(tx *bstore.Tx) error {
+		exporttx(tx, mbox, dst, accountDir, mailbox)
+		return nil
+	})
+	xcheckf(err, "transaction")
+}
+
+func exporttx(tx *bstore.Tx, mbox bool, dst, accountDir, mailbox string) {
+	id2name := map[int64]string{}
+	name2id := map[string]int64{}
+
+	mailboxes, err := bstore.QueryTx[store.Mailbox](tx).List()
+	xcheckf(err, "query mailboxes")
+	for _, mb := range mailboxes {
+		id2name[mb.ID] = mb.Name
+		name2id[mb.Name] = mb.ID
+	}
+
+	var mailboxID int64
+	if mailbox != "" {
+		var ok bool
+		mailboxID, ok = name2id[mailbox]
+		if !ok {
+			log.Fatalf("mailbox %q not found", mailbox)
+		}
+	}
+
+	mboxes := map[string]*os.File{}
+
+	// Open mbox files or create dirs.
+	var names []string
+	for _, name := range id2name {
+		if mailbox != "" && name != mailbox {
+			continue
+		}
+		names = append(names, name)
+	}
+	// We need to sort the names because maildirs can create subdirs. Ranging over
+	// id2name directly would randomize the directory names, we would create a sub
+	// maildir before the parent, and fail with "dir exists" when creating the parent
+	// dir.
+	sort.Slice(names, func(i, j int) bool {
+		return names[i] < names[j]
+	})
+	for _, name := range names {
+		p := dst
+		if mailbox == "" {
+			p = filepath.Join(p, name)
+		}
+
+		os.MkdirAll(filepath.Dir(p), 0770)
+		if mbox {
+			mbp := p
+			if mailbox == "" {
+				mbp += ".mbox"
+			}
+			f, err := os.OpenFile(mbp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
+			xcheckf(err, "creating mbox file")
+			log.Printf("creating mbox file %s", mbp)
+			mboxes[name] = f
+		} else {
+			err = os.Mkdir(p, 0770)
+			xcheckf(err, "making maildir")
+			log.Printf("creating maildir %s", p)
+			subdirs := []string{"new", "cur", "tmp"}
+			for _, subdir := range subdirs {
+				err = os.Mkdir(filepath.Join(p, subdir), 0770)
+				xcheckf(err, "making maildir subdir")
+			}
+		}
+	}
+
+	q := bstore.QueryTx[store.Message](tx)
+	if mailboxID > 0 {
+		q.FilterNonzero(store.Message{MailboxID: mailboxID})
+	}
+	defer q.Close()
+	for {
+		m, err := q.Next()
+		if err == bstore.ErrAbsent {
+			break
+		}
+		xcheckf(err, "next message")
+
+		mbname := id2name[m.MailboxID]
+
+		p := dst
+		if mailbox == "" {
+			p = filepath.Join(p, mbname)
+		}
+
+		mp := filepath.Join(accountDir, "msg", store.MessagePath(m.ID))
+		var mr io.ReadCloser
+		if m.Size == int64(len(m.MsgPrefix)) {
+			log.Printf("message size is prefix size for m id %d", m.ID)
+			mr = io.NopCloser(bytes.NewReader(m.MsgPrefix))
+		} else {
+			mpf, err := os.Open(mp)
+			xcheckf(err, "open message file")
+			st, err := mpf.Stat()
+			xcheckf(err, "stat message file")
+			size := st.Size() + int64(len(m.MsgPrefix))
+			if size != m.Size {
+				log.Fatalf("message size mismatch, database has %d, size is %d+%d=%d", m.Size, len(m.MsgPrefix), st.Size(), size)
+			}
+			mr = store.FileMsgReader(m.MsgPrefix, mpf)
+		}
+
+		if mbox {
+			// todo: should we put status flags in Status or X-Status header inside the message?
+			// todo: should we do anything with Content-Length headers? changing the escaping could invalidate those. is anything checking that field?
+
+			f := mboxes[mbname]
+			mailfrom := "mox"
+			if m.MailFrom != "" {
+				mailfrom = m.MailFrom
+			}
+			_, err := fmt.Fprintf(f, "From %s %s\n", mailfrom, m.Received.Format(time.ANSIC))
+			xcheckf(err, "writing from header")
+			r := bufio.NewReader(mr)
+			for {
+				line, rerr := r.ReadBytes('\n')
+				if rerr != io.EOF {
+					xcheckf(rerr, "reading from message")
+				}
+				if len(line) > 0 {
+					if bytes.HasSuffix(line, []byte("\r\n")) {
+						line = line[:len(line)-1]
+						line[len(line)-1] = '\n'
+					}
+					if bytes.HasPrefix(bytes.TrimLeft(line, ">"), []byte("From ")) {
+						_, err = fmt.Fprint(f, ">")
+						xcheckf(err, "writing escaping >")
+					}
+					_, err = f.Write(line)
+					xcheckf(err, "writing line")
+				}
+				if rerr == io.EOF {
+					break
+				}
+			}
+			_, err = fmt.Fprint(f, "\n")
+			xcheckf(err, "writing end of message newline")
+		} else {
+			if m.Flags.Seen {
+				p = filepath.Join(p, "cur")
+			} else {
+				p = filepath.Join(p, "new")
+			}
+			name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID)
+			// todo: more flags? forwarded, (non)junk, phishing, mdnsent would be nice. but what is the convention. dovecot-keywords sounds non-standard.
+			if m.Flags.Seen {
+				name += "S"
+			}
+			if m.Flags.Answered {
+				name += "R"
+			}
+			if m.Flags.Flagged {
+				name += "F"
+			}
+			if m.Flags.Draft {
+				name += "D"
+			}
+			p = filepath.Join(p, name)
+			f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
+			xcheckf(err, "creating message file in maildir")
+
+			r := bufio.NewReader(mr)
+			for {
+				line, rerr := r.ReadBytes('\n')
+				if rerr != io.EOF {
+					xcheckf(rerr, "reading from message")
+				}
+				if len(line) > 0 {
+					if bytes.HasSuffix(line, []byte("\r\n")) {
+						line = line[:len(line)-1]
+						line[len(line)-1] = '\n'
+					}
+					_, err = f.Write(line)
+					xcheckf(err, "writing line")
+				}
+				if rerr == io.EOF {
+					break
+				}
+			}
+			mr.Close()
+			err = f.Close()
+			xcheckf(err, "closing new file in maildir")
+		}
+
+		mr.Close()
+	}
+
+	if mbox {
+		for _, f := range mboxes {
+			err = f.Close()
+			xcheckf(err, "closing mbox file")
+		}
+	}
+}

gendoc.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+
+(
+cat <<EOF
+/*
+Command mox is a modern full-featured open source secure mail server for
+low-maintenance self-hosted email.
+
+  - Quick and easy to set up with quickstart and automatic TLS with ACME and
+    Let's Encrypt.
+  - IMAP4 with extensions for accessing email.
+  - SMTP with SPF, DKIM, DMARC, DNSBL, MTA-STS, TLSRPT for exchanging email.
+  - Reputation-based and content-based spam filtering.
+  - Internationalized email.
+  - Admin web interface.
+
+# Commands
+
+EOF
+
+./mox 2>&1 | sed 's/^\( *\|usage: \)/\t/'
+
+cat <<EOF
+
+Many commands talk to a running mox instance, through the ctl file in the data
+directory. Specify the configuration file (that holds the path to the data
+directory) through the -config flag or MOXCONF environment variable.
+
+EOF
+
+./mox helpall 2>&1
+
+cat <<EOF
+*/
+package main
+
+// NOTE: DO NOT EDIT, this file is generated by gendoc.sh.
+EOF
+)>doc.go
+gofmt -w doc.go
+
+(
+cat <<EOF
+/*
+Package config holds the configuration file definitions for mox.conf (Static)
+and domains.conf (Dynamic).
+
+Annotated empty/default configuration files you could use as a starting point
+for your mox.conf and domains.conf, as generated by "mox config
+describe-static" and "mox config describe-domains":
+
+# mox.conf
+
+EOF
+./mox config describe-static | sed 's/^/\t/'
+
+cat <<EOF
+
+# domains.conf
+
+EOF
+./mox config describe-domains | sed 's/^/\t/'
+
+cat <<EOF
+*/
+package config
+
+// NOTE: DO NOT EDIT, this file is generated by ../gendoc.sh.
+EOF
+)>config/doc.go
+gofmt -w config/doc.go

go.mod

@@ -0,0 +1,31 @@
+module github.com/mjl-/mox
+
+go 1.18
+
+require (
+	github.com/mjl-/bstore v0.0.0-20230114150735-9d9c0a2dcc79
+	github.com/mjl-/sconf v0.0.4
+	github.com/mjl-/sherpa v0.6.5
+	github.com/mjl-/sherpadoc v0.0.10
+	github.com/mjl-/sherpaprom v0.0.2
+	github.com/prometheus/client_golang v1.14.0
+	golang.org/x/crypto v0.5.0
+	golang.org/x/net v0.5.0
+	golang.org/x/text v0.6.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mjl-/xfmt v0.0.0-20190521151243-39d9c00752ce // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/tools v0.1.12 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+)

go.sum

@@ -0,0 +1,507 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mjl-/bstore v0.0.0-20230114150735-9d9c0a2dcc79 h1:bptDsTAvgtmIOrhKjMVrUm4JBkF0jekpVmsZdkgALPM=
+github.com/mjl-/bstore v0.0.0-20230114150735-9d9c0a2dcc79/go.mod h1:/cD25FNBaDfvL/plFRxI3Ba3E+wcB0XVOS8nJDqndg0=
+github.com/mjl-/sconf v0.0.4 h1:uyfn4vv5qOULSgiwQsPbbgkiONKnMFMsSOhsHfAiYwI=
+github.com/mjl-/sconf v0.0.4/go.mod h1:ezf7YOn7gtClo8y71SqgZKaEkyMQ5Te7vkv4PmTTfwM=
+github.com/mjl-/sherpa v0.6.5 h1:d90uG/j8fw+2M+ohCTAcVwTSUURGm8ktYDScJO1nKog=
+github.com/mjl-/sherpa v0.6.5/go.mod h1:dSpAOdgpwdqQZ72O4n3EHo/tR68eKyan8tYYraUMPNc=
+github.com/mjl-/sherpadoc v0.0.0-20190505200843-c0a7f43f5f1d/go.mod h1:5khTKxoKKNXcB8bkVUO6GlzC7PFtMmkHq578lPbmnok=
+github.com/mjl-/sherpadoc v0.0.10 h1:tvRVd37IIGg70ZmNkNKNnjDSPtKI5/DdEIukMkWtZYE=
+github.com/mjl-/sherpadoc v0.0.10/go.mod h1:vh5zcsk3j/Tvm725EY+unTZb3EZcZcpiEQzrODSa6+I=
+github.com/mjl-/sherpaprom v0.0.2 h1:1dlbkScsNafM5jURI44uiWrZMSwfZtcOFEEq7vx2C1Y=
+github.com/mjl-/sherpaprom v0.0.2/go.mod h1:cl5nMNOvqhzMiQJ2FzccQ9ReivjHXe53JhOVkPfSvw4=
+github.com/mjl-/xfmt v0.0.0-20190521151243-39d9c00752ce h1:oyFmIHo3GLWZzb0odAzN9QUy0MTW6P8JaNRnNVGCBCk=
+github.com/mjl-/xfmt v0.0.0-20190521151243-39d9c00752ce/go.mod h1:DIEOLmETMQHHr4OgwPG7iC37rDiN9MaZIZxNm5hBtL8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190503130316-740c07785007/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

http/account.go

@@ -0,0 +1,114 @@
+package http
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+
+	_ "embed"
+
+	"github.com/mjl-/sherpa"
+	"github.com/mjl-/sherpaprom"
+
+	"github.com/mjl-/mox/metrics"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/store"
+)
+
+//go:embed accountapi.json
+var accountapiJSON []byte
+
+//go:embed account.html
+var accountHTML []byte
+
+var accountDoc = mustParseAPI(accountapiJSON)
+
+var accountSherpaHandler http.Handler
+
+func init() {
+	collector, err := sherpaprom.NewCollector("moxaccount", nil)
+	if err != nil {
+		xlog.Fatalx("creating sherpa prometheus collector", err)
+	}
+
+	accountSherpaHandler, err = sherpa.NewHandler("/account/api/", moxvar.Version, Account{}, &accountDoc, &sherpa.HandlerOpts{Collector: collector, AdjustFunctionNames: "none"})
+	if err != nil {
+		xlog.Fatalx("sherpa handler", err)
+	}
+}
+
+// Account exports web API functions for the account web interface. All its
+// methods are exported under /account/api/. Function calls require valid HTTP
+// Authentication credentials of a user.
+type Account struct{}
+
+func accountHandle(w http.ResponseWriter, r *http.Request) {
+	ctx := context.WithValue(r.Context(), mlog.CidKey, mox.Cid())
+	log := xlog.WithContext(ctx).Fields(mlog.Field("userauth", ""))
+	var accountName string
+	authResult := "error"
+	defer func() {
+		metrics.AuthenticationInc("httpaccount", "httpbasic", authResult)
+	}()
+	// todo: should probably add a cache here instead of looking up password in database all the time, just like in admin.go
+	if auth := r.Header.Get("Authorization"); auth == "" || !strings.HasPrefix(auth, "Basic ") {
+	} else if authBuf, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")); err != nil {
+		log.Infox("parsing base64", err)
+	} else if t := strings.SplitN(string(authBuf), ":", 2); len(t) != 2 {
+		log.Info("bad user:pass form")
+	} else if acc, err := store.OpenEmailAuth(t[0], t[1]); err != nil {
+		if errors.Is(err, store.ErrUnknownCredentials) {
+			authResult = "badcreds"
+		}
+		log.Infox("open account", err)
+	} else {
+		accountName = acc.Name
+		authResult = "ok"
+	}
+	if accountName == "" {
+		w.Header().Set("WWW-Authenticate", `Basic realm="mox account - login with email address and password"`)
+		w.WriteHeader(http.StatusUnauthorized)
+		fmt.Fprintln(w, "http 401 - unauthorized - mox account - login with email address and password")
+		return
+	}
+
+	if r.Method == "GET" && r.URL.Path == "/account/" {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.Header().Set("Cache-Control", "no-cache; max-age=0")
+		f, err := os.Open("http/account.html")
+		if err == nil {
+			defer f.Close()
+			io.Copy(w, f)
+		} else {
+			w.Write(accountHTML)
+		}
+		return
+	}
+	accountSherpaHandler.ServeHTTP(w, r.WithContext(context.WithValue(ctx, authCtxKey, accountName)))
+}
+
+type ctxKey string
+
+var authCtxKey ctxKey = "account"
+
+// SetPassword saves a new password for the account, invalidating the previous password.
+// Sessions are not interrupted, and will keep working. New login attempts must use the new password.
+// Password must be at least 8 characters.
+func (Account) SetPassword(ctx context.Context, password string) {
+	if len(password) < 8 {
+		panic(&sherpa.Error{Code: "user:error", Message: "password must be at least 8 characters"})
+	}
+	accountName := ctx.Value(authCtxKey).(string)
+	acc, err := store.OpenAccount(accountName)
+	xcheckf(ctx, err, "open account")
+	defer acc.Close()
+	err = acc.SetPassword(password)
+	xcheckf(ctx, err, "setting password")
+}

http/account.html

@@ -0,0 +1,214 @@
+<!doctype html>
+<html>
+	<head>
+		<title>Mox Account</title>
+		<meta charset="utf-8" />
+		<meta name="viewport" content="width=device-width, initial-scale=1" />
+		<style>
+body, html { padding: 1em; font-size: 16px; }
+* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }
+h1, h2, h3, h4 { margin-bottom: 1ex; }
+h1 { font-size: 1.2rem; }
+h2 { font-size: 1.1rem; }
+h3, h4 { font-size: 1rem; }
+.literal { background-color: #fdfdfd; padding: .5em 1em; border: 1px solid #eee; border-radius: 4px; white-space: pre-wrap; font-family: mono; font-size: 15px; tab-size: 4; }
+table td, table th { padding: .2em .5em; }
+table > tbody > tr:nth-child(odd) { background-color: #f8f8f8; }
+p { margin-bottom: 1em; max-width: 50em; }
+[title] { text-decoration: underline; text-decoration-style: dotted; }
+fieldset { border: 0; }
+#page { opacity: 1; animation: fadein 0.15s ease-in; }
+#page.loading { opacity: 0.1; animation: fadeout 1s ease-out; }
+@keyframes fadein { 0% { opacity: 0 } 100% { opacity: 1 } }
+@keyframes fadeout { 0% { opacity: 1 } 100% { opacity: 0.1 } }
+		</style>
+		<script src="api/sherpa.js"></script>
+	</head>
+	<body>
+		<div id="page">Loading...</div>
+
+		<script>
+const [dom, style, attr, prop] = (function() {
+function _domKids(e, ...kl) {
+	kl.forEach(k => {
+		if (typeof k === 'string' || k instanceof String) {
+			e.appendChild(document.createTextNode(k))
+		} else if (k instanceof Node) {
+			e.appendChild(k)
+		} else if (Array.isArray(k)) {
+			_domKids(e, ...k)
+		} else if (typeof k === 'function') {
+			if (!k.name) {
+				throw new Error('function without name', k)
+			}
+			e.addEventListener(k.name, k)
+		} else if (typeof k === 'object' && k !== null) {
+			if (k.root) {
+				e.appendChild(k.root)
+				return
+			}
+			for (const key in k) {
+				const value = k[key]
+				if (key === '_prop') {
+					for (const prop in value) {
+						e[prop] = value[prop]
+					}
+				} else if (key === '_attr') {
+					for (const prop in value) {
+						e.setAttribute(prop, value[prop])
+					}
+				} else if (key === '_listen') {
+					e.addEventListener(...value)
+				} else {
+					e.style[key] = value
+				}
+			}
+		} else {
+			console.log('bad kid', k)
+			throw new Error('bad kid')
+		}
+	})
+}
+const _dom = (kind, ...kl) => {
+	const t = kind.split('.')
+	const e = document.createElement(t[0])
+	for (let i = 1; i < t.length; i++) {
+		e.classList.add(t[i])
+	}
+	_domKids(e, kl)
+	return e
+}
+_dom._kids = function(e, ...kl) {
+	while(e.firstChild) {
+		e.removeChild(e.firstChild)
+	}
+	_domKids(e, kl)
+}
+const dom = new Proxy(_dom, {
+	get: function(dom, prop) {
+		if (prop in dom) {
+			return dom[prop]
+		}
+		const fn = (...kl) => _dom(prop, kl)
+		dom[prop] = fn
+		return fn
+	},
+	apply: function(target, that, args) {
+		if (args.length === 1 && typeof args[0] === 'object' && !Array.isArray(args[0])) {
+			return {_attr: args[0]}
+		}
+		return _dom(...args)
+	},
+})
+const style = x => x
+const attr = x => { return {_attr: x} }
+const prop = x => { return {_prop: x} }
+return [dom, style, attr, prop]
+})()
+
+const tr = dom.tr
+const td = dom.td
+const th = dom.th
+
+const crumblink = (text, link) => dom.a(text, attr({href: link}))
+const crumbs = (...l) => [dom.h1(l.map((e, index) => index === 0 ? e : [' / ', e])), dom.br()]
+
+const footer = dom.div(
+	style({marginTop: '6ex', opacity: 0.75}),
+	dom.a(attr({href: 'https://github.com/mjl-/mox'}), 'mox'),
+	' ',
+	api._sherpa.version,
+)
+
+const index = async () => {
+	let form, fieldset, password1, password2
+
+	const blockStyle = style({
+		display: 'block',
+		marginBottom: '1ex',
+	})
+
+	const page = document.getElementById('page')
+	dom._kids(page,
+		crumbs('Mox Account'),
+		dom.h2('Change password'),
+		form=dom.form(
+			fieldset=dom.fieldset(
+				dom.label(
+					style({display: 'inline-block'}),
+					'New password',
+					dom.br(),
+					password1=dom.input(attr({type: 'password', required: ''})),
+				),
+				' ',
+				dom.label(
+					style({display: 'inline-block'}),
+					'New password repeat',
+					dom.br(),
+					password2=dom.input(attr({type: 'password', required: ''})),
+				),
+				' ',
+				dom.button('Change password'),
+			),
+			async function submit(e) {
+				e.stopPropagation()
+				e.preventDefault()
+				if (!password1.value || password1.value !== password2.value) {
+					window.alert('Passwords do not match.')
+					return
+				}
+				fieldset.disabled = true
+				try {
+					await api.SetPassword(password1.value)
+					window.alert('Password has been changed.')
+					form.reset()
+				} catch (err) {
+					console.log('error', err)
+					window.alert('Error: ' + err.message)
+				} finally {
+					fieldset.disabled = false
+				}
+			},
+		),
+		footer,
+	)
+}
+
+const init = async () => {
+	let curhash
+
+	const page = document.getElementById('page')
+
+	const hashChange = async () => {
+		if (curhash === window.location.hash) {
+			return
+		}
+		let h = window.location.hash
+		if (h !== '' && h.substring(0, 1) == '#') {
+			h = h.substring(1)
+		}
+		page.classList.add('loading')
+		try {
+			if (h == '') {
+				await index()
+			} else {
+				dom._kids(page, 'page not found')
+			}
+		} catch (err) {
+			console.log('error', err)
+			window.alert('Error: ' + err.message)
+			window.location.hash = curhash
+			curhash = window.location.hash
+			return
+		}
+		curhash = window.location.hash
+		page.classList.remove('loading')
+	}
+	window.addEventListener('hashchange', hashChange)
+	hashChange()
+}
+
+window.addEventListener('load', init)
+		</script>
+	</body>
+</html>

http/account_test.go

@@ -0,0 +1,3 @@
+package http
+
+// todo: write test for account api calls, at least for authentation and SetPassword.

http/accountapi.json

@@ -0,0 +1,25 @@
+{
+	"Name": "Account",
+	"Docs": "Account exports web API functions for the account web interface. All its\nmethods are exported under /account/api/. Function calls require valid HTTP\nAuthentication credentials of a user.",
+	"Functions": [
+		{
+			"Name": "SetPassword",
+			"Docs": "SetPassword saves a new password for the account, invalidating the previous password.\nSessions are not interrupted, and will keep working. New login attempts must use the new password.\nPassword must be at least 8 characters.",
+			"Params": [
+				{
+					"Name": "password",
+					"Typewords": [
+						"string"
+					]
+				}
+			],
+			"Returns": []
+		}
+	],
+	"Sections": [],
+	"Structs": [],
+	"Ints": [],
+	"Strings": [],
+	"SherpaVersion": 0,
+	"SherpadocVersion": 1
+}

http/admin_test.go

@@ -0,0 +1,123 @@
+package http
+
+import (
+	"context"
+	"crypto/ed25519"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"golang.org/x/crypto/bcrypt"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mox-"
+)
+
+func TestAdminAuth(t *testing.T) {
+	test := func(passwordfile, authHdr string, expect bool) {
+		t.Helper()
+
+		ok := checkAdminAuth(context.Background(), passwordfile, authHdr)
+		if ok != expect {
+			t.Fatalf("got %v, expected %v", ok, expect)
+		}
+	}
+
+	const authOK = "Basic YWRtaW46bW94dGVzdDEyMw=="  // admin:moxtest123
+	const authBad = "Basic YWRtaW46YmFkcGFzc3dvcmQ=" // admin:badpassword
+
+	const path = "../testdata/http-passwordfile"
+	os.Remove(path)
+	defer os.Remove(path)
+
+	test(path, authOK, false) // Password file does not exist.
+
+	adminpwhash, err := bcrypt.GenerateFromPassword([]byte("moxtest123"), bcrypt.DefaultCost)
+	if err != nil {
+		t.Fatalf("generate bcrypt hash: %v", err)
+	}
+	if err := os.WriteFile(path, adminpwhash, 0660); err != nil {
+		t.Fatalf("write password file: %v", err)
+	}
+	// We loop to also exercise the auth cache.
+	for i := 0; i < 2; i++ {
+		test(path, "", false)                 // Empty/missing header.
+		test(path, "Malformed ", false)       // Not "Basic"
+		test(path, "Basic malformed ", false) // Bad base64.
+		test(path, "Basic dGVzdA== ", false)  // base64 is ok, but wrong tokens inside.
+		test(path, authBad, false)            // Wrong password.
+		test(path, authOK, true)
+	}
+}
+
+func TestCheckDomain(t *testing.T) {
+	// NOTE: we aren't currently looking at the results, having the code paths executed is better than nothing.
+
+	resolver := dns.MockResolver{
+		MX: map[string][]*net.MX{
+			"mox.example.": {{Host: "mail.mox.example.", Pref: 10}},
+		},
+		A: map[string][]string{
+			"mail.mox.example.": {"127.0.0.2"},
+		},
+		AAAA: map[string][]string{
+			"mail.mox.example.": {"127.0.0.2"},
+		},
+		TXT: map[string][]string{
+			"mox.example.":                 {"v=spf1 mx -all"},
+			"test._domainkey.mox.example.": {"v=DKIM1;h=sha256;k=ed25519;p=ln5zd/JEX4Jy60WAhUOv33IYm2YZMyTQAdr9stML504="},
+			"_dmarc.mox.example.":          {"v=DMARC1; p=reject; rua=mailto:mjl@mox.example"},
+			"_smtp._tls.mox.example":       {"v=TLSRPTv1; rua=mailto:tlsrpt@mox.example;"},
+			"_mta-sts.mox.example":         {"v=STSv1; id=20160831085700Z"},
+		},
+		CNAME: map[string]string{},
+	}
+
+	listener := config.Listener{
+		IPs:            []string{"127.0.0.2"},
+		Hostname:       "mox.example",
+		HostnameDomain: dns.Domain{ASCII: "mox.example"},
+	}
+	listener.SMTP.Enabled = true
+	listener.AutoconfigHTTPS.Enabled = true
+	listener.MTASTSHTTPS.Enabled = true
+
+	mox.Conf.Static.Listeners = map[string]config.Listener{
+		"public": listener,
+	}
+	domain := config.Domain{
+		DKIM: config.DKIM{
+			Selectors: map[string]config.Selector{
+				"test": {
+					HashEffective:    "sha256",
+					HeadersEffective: []string{"From", "Date", "Subject"},
+					Key:              ed25519.NewKeyFromSeed(make([]byte, 32)), // warning: fake zero key, do not copy this code.
+					Domain:           dns.Domain{ASCII: "test"},
+				},
+				"missing": {
+					HashEffective:    "sha256",
+					HeadersEffective: []string{"From", "Date", "Subject"},
+					Key:              ed25519.NewKeyFromSeed(make([]byte, 32)), // warning: fake zero key, do not copy this code.
+					Domain:           dns.Domain{ASCII: "missing"},
+				},
+			},
+			Sign: []string{"test", "test2"},
+		},
+	}
+	mox.Conf.Dynamic.Domains = map[string]config.Domain{
+		"mox.example": domain,
+	}
+
+	// Make a dialer that fails immediately before actually connecting.
+	done := make(chan struct{})
+	close(done)
+	dialer := &net.Dialer{Deadline: time.Now().Add(-time.Second), Cancel: done}
+
+	checkDomain(context.Background(), resolver, dialer, "mox.example")
+	// todo: check returned data
+
+	Admin{}.Domains(context.Background())        // todo: check results
+	dnsblsStatus(context.Background(), resolver) // todo: check results
+}

http/autoconf.go

@@ -0,0 +1,344 @@
+package http
+
+import (
+	"encoding/xml"
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/smtp"
+)
+
+var (
+	metricAutoconf = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mox_autoconf_request_total",
+			Help: "Number of autoconf requests.",
+		},
+		[]string{"domain"},
+	)
+	metricAutodiscover = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mox_autodiscover_request_total",
+			Help: "Number of autodiscover requests.",
+		},
+		[]string{"domain"},
+	)
+)
+
+// Autoconfiguration/Autodiscovery:
+//
+//   - Thunderbird will request an "autoconfig" xml file.
+//   - Microsoft tools will request an "autodiscovery" xml file.
+//   - In my tests on an internal domain, iOS mail only talks to Apple servers, then
+//   does not attempt autoconfiguration. Possibly due to them being private DNS names.
+//
+// DNS records seem optional, but autoconfig.<domain> and autodiscover.<domain>
+// (both CNAME or A) are useful, and so is SRV _autodiscovery._tcp.<domain> 0 0 443
+// autodiscover.<domain> (or just <hostname> directly).
+//
+// Autoconf/discovery only works with valid TLS certificates, not with self-signed
+// certs. So use it on public endpoints with certs signed by common CA's, or run
+// your own (internal) CA and import the CA cert on your devices.
+//
+// Also see https://roll.urown.net/server/mail/autoconfig.html
+
+// Autoconfiguration for Mozilla Thunderbird.
+// User should create a DNS record: autoconfig.<domain> (CNAME or A).
+// See https://wiki.mozilla.org/Thunderbird:Autoconfiguration:ConfigFileFormat
+func autoconfHandle(l config.Listener) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log := xlog.WithContext(r.Context())
+
+		var addrDom string
+		defer func() {
+			metricAutoconf.WithLabelValues(addrDom).Inc()
+		}()
+
+		email := r.FormValue("emailaddress")
+		log.Debug("autoconfig request", mlog.Field("email", email))
+		addr, err := smtp.ParseAddress(email)
+		if err != nil {
+			http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest)
+			return
+		}
+
+		if _, ok := mox.Conf.Domain(addr.Domain); !ok {
+			http.Error(w, "400 - bad request - unknown domain", http.StatusBadRequest)
+			return
+		}
+		addrDom = addr.Domain.Name()
+
+		hostname := l.HostnameDomain
+		if hostname.IsZero() {
+			hostname = mox.Conf.Static.HostnameDomain
+		}
+
+		// Thunderbird doesn't seem to allow U-labels, always return ASCII names.
+		var resp autoconfigResponse
+		resp.Version = "1.1"
+		resp.EmailProvider.ID = addr.Domain.ASCII
+		resp.EmailProvider.Domain = addr.Domain.ASCII
+		resp.EmailProvider.DisplayName = email
+		resp.EmailProvider.DisplayShortName = addr.Domain.ASCII
+
+		var imapPort int
+		var imapSocket string
+		if l.IMAPS.Enabled {
+			imapPort = config.Port(l.IMAPS.Port, 993)
+			imapSocket = "SSL"
+		} else if l.IMAP.Enabled {
+			imapPort = config.Port(l.IMAP.Port, 143)
+			if l.TLS != nil {
+				imapSocket = "STARTTLS"
+			} else {
+				imapSocket = "plain"
+			}
+		} else {
+			log.Error("autoconfig: no imap configured?")
+		}
+
+		// todo: specify SCRAM-SHA256 once thunderbird and autoconfig supports it. we could implement CRAM-MD5 and use it.
+
+		resp.EmailProvider.IncomingServer.Type = "imap"
+		resp.EmailProvider.IncomingServer.Hostname = hostname.ASCII
+		resp.EmailProvider.IncomingServer.Port = imapPort
+		resp.EmailProvider.IncomingServer.SocketType = imapSocket
+		resp.EmailProvider.IncomingServer.Username = email
+		resp.EmailProvider.IncomingServer.Authentication = "password-cleartext"
+
+		var smtpPort int
+		var smtpSocket string
+		if l.Submissions.Enabled {
+			smtpPort = config.Port(l.Submissions.Port, 465)
+			smtpSocket = "SSL"
+		} else if l.Submission.Enabled {
+			smtpPort = config.Port(l.Submission.Port, 587)
+			if l.TLS != nil {
+				smtpSocket = "STARTTLS"
+			} else {
+				smtpSocket = "plain"
+			}
+		} else {
+			log.Error("autoconfig: no smtp submission configured?")
+		}
+
+		resp.EmailProvider.OutgoingServer.Type = "smtp"
+		resp.EmailProvider.OutgoingServer.Hostname = hostname.ASCII
+		resp.EmailProvider.OutgoingServer.Port = smtpPort
+		resp.EmailProvider.OutgoingServer.SocketType = smtpSocket
+		resp.EmailProvider.OutgoingServer.Username = email
+		resp.EmailProvider.OutgoingServer.Authentication = "password-cleartext"
+
+		// todo: should we put the email address in the URL?
+		resp.ClientConfigUpdate.URL = fmt.Sprintf("https://%s/mail/config-v1.1.xml", hostname.ASCII)
+
+		w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+		enc := xml.NewEncoder(w)
+		enc.Indent("", "\t")
+		fmt.Fprint(w, xml.Header)
+		if err := enc.Encode(resp); err != nil {
+			log.Errorx("marshal autoconfig response", err)
+		}
+	}
+}
+
+// Autodiscover from Microsoft, also used by Thunderbird.
+// User should create a DNS record: _autodiscover._tcp.<domain> IN SRV 0 0 443 <hostname or autodiscover.<domain>>
+func autodiscoverHandle(l config.Listener) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log := xlog.WithContext(r.Context())
+
+		var addrDom string
+		defer func() {
+			metricAutodiscover.WithLabelValues(addrDom).Inc()
+		}()
+
+		if r.Method != "POST" {
+			http.Error(w, "405 - method not allowed - post required", http.StatusMethodNotAllowed)
+			return
+		}
+
+		var req autodiscoverRequest
+		if err := xml.NewDecoder(r.Body).Decode(&req); err != nil {
+			http.Error(w, "400 - bad request - parsing autodiscover request: "+err.Error(), http.StatusMethodNotAllowed)
+			return
+		}
+
+		log.Debug("autodiscover request", mlog.Field("email", req.Request.EmailAddress))
+
+		addr, err := smtp.ParseAddress(req.Request.EmailAddress)
+		if err != nil {
+			http.Error(w, "400 - bad request - invalid parameter emailaddress", http.StatusBadRequest)
+			return
+		}
+
+		if _, ok := mox.Conf.Domain(addr.Domain); !ok {
+			http.Error(w, "400 - bad request - unknown domain", http.StatusBadRequest)
+			return
+		}
+		addrDom = addr.Domain.Name()
+
+		hostname := l.HostnameDomain
+		if hostname.IsZero() {
+			hostname = mox.Conf.Static.HostnameDomain
+		}
+
+		// The docs are generated and fragmented in many tiny pages, hard to follow.
+		// High-level starting point, https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxdscli/78530279-d042-4eb0-a1f4-03b18143cd19
+		// Request: https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxdscli/2096fab2-9c3c-40b9-b123-edf6e8d55a9b
+		// Response, protocol: https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxdscli/f4238db6-a983-435c-807a-b4b4a624c65b
+		// It appears autodiscover does not allow specifying SCRAM-SHA256 as authentication method. See https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxdscli/21fd2dd5-c4ee-485b-94fb-e7db5da93726
+
+		var imapPort int
+		imapSSL := "off"
+		var imapEncryption string
+		if l.IMAPS.Enabled {
+			imapPort = config.Port(l.IMAPS.Port, 993)
+			imapSSL = "on"
+			imapEncryption = "TLS" // Assuming this means direct TLS.
+		} else if l.IMAP.Enabled {
+			imapPort = config.Port(l.IMAP.Port, 143)
+			if l.TLS != nil {
+				imapSSL = "on"
+			}
+		} else {
+			log.Error("autoconfig: no imap configured?")
+		}
+
+		var smtpPort int
+		smtpSSL := "off"
+		var smtpEncryption string
+		if l.Submissions.Enabled {
+			smtpPort = config.Port(l.Submissions.Port, 465)
+			smtpSSL = "on"
+			smtpEncryption = "TLS" // Assuming this means direct TLS.
+		} else if l.Submission.Enabled {
+			smtpPort = config.Port(l.Submission.Port, 587)
+			if l.TLS != nil {
+				smtpSSL = "on"
+			}
+		} else {
+			log.Error("autoconfig: no smtp submission configured?")
+		}
+
+		w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+
+		resp := autodiscoverResponse{}
+		resp.XMLName.Local = "Autodiscover"
+		resp.XMLName.Space = "http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006"
+		resp.Response.XMLName.Local = "Response"
+		resp.Response.XMLName.Space = "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"
+		resp.Response.Account = autodiscoverAccount{
+			AccountType: "email",
+			Action:      "settings",
+			Protocol: []autodiscoverProtocol{
+				{
+					Type:         "IMAP",
+					Server:       hostname.ASCII,
+					Port:         imapPort,
+					LoginName:    req.Request.EmailAddress,
+					SSL:          imapSSL,
+					Encryption:   imapEncryption,
+					SPA:          "off", // Override default "on", this is Microsofts proprietary authentication protocol.
+					AuthRequired: "on",
+				},
+				{
+					Type:         "SMTP",
+					Server:       hostname.ASCII,
+					Port:         smtpPort,
+					LoginName:    req.Request.EmailAddress,
+					SSL:          smtpSSL,
+					Encryption:   smtpEncryption,
+					SPA:          "off", // Override default "on", this is Microsofts proprietary authentication protocol.
+					AuthRequired: "on",
+				},
+			},
+		}
+		enc := xml.NewEncoder(w)
+		enc.Indent("", "\t")
+		fmt.Fprint(w, xml.Header)
+		if err := enc.Encode(resp); err != nil {
+			log.Errorx("marshal autodiscover response", err)
+		}
+	}
+}
+
+// Thunderbird requests these URLs for autoconfig/autodiscover:
+// https://autoconfig.example.org/mail/config-v1.1.xml?emailaddress=user%40example.org
+// https://autodiscover.example.org/autodiscover/autodiscover.xml
+// https://example.org/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress=user%40example.org
+// https://example.org/autodiscover/autodiscover.xml
+type autoconfigResponse struct {
+	XMLName xml.Name `xml:"clientConfig"`
+	Version string   `xml:"version,attr"`
+
+	EmailProvider struct {
+		ID               string `xml:"id,attr"`
+		Domain           string `xml:"domain"`
+		DisplayName      string `xml:"displayName"`
+		DisplayShortName string `xml:"displayShortName"`
+
+		IncomingServer struct {
+			Type           string `xml:"type,attr"`
+			Hostname       string `xml:"hostname"`
+			Port           int    `xml:"port"`
+			SocketType     string `xml:"socketType"`
+			Username       string `xml:"username"`
+			Authentication string `xml:"authentication"`
+		} `xml:"incomingServer"`
+
+		OutgoingServer struct {
+			Type           string `xml:"type,attr"`
+			Hostname       string `xml:"hostname"`
+			Port           int    `xml:"port"`
+			SocketType     string `xml:"socketType"`
+			Username       string `xml:"username"`
+			Authentication string `xml:"authentication"`
+		} `xml:"outgoingServer"`
+	} `xml:"emailProvider"`
+
+	ClientConfigUpdate struct {
+		URL string `xml:"url,attr"`
+	} `xml:"clientConfigUpdate"`
+}
+
+type autodiscoverRequest struct {
+	XMLName xml.Name `xml:"Autodiscover"`
+	Request struct {
+		EmailAddress             string `xml:"EMailAddress"`
+		AcceptableResponseSchema string `xml:"AcceptableResponseSchema"`
+	}
+}
+
+type autodiscoverResponse struct {
+	XMLName  xml.Name
+	Response struct {
+		XMLName xml.Name
+		Account autodiscoverAccount
+	}
+}
+
+type autodiscoverAccount struct {
+	AccountType string
+	Action      string
+	Protocol    []autodiscoverProtocol
+}
+
+type autodiscoverProtocol struct {
+	Type          string
+	Server        string
+	Port          int
+	DirectoryPort int
+	ReferralPort  int
+	LoginName     string
+	SSL           string
+	Encryption    string `xml:",omitempty"`
+	SPA           string
+	AuthRequired  string
+}

http/autoconf_test.go

@@ -0,0 +1,26 @@
+package http
+
+import (
+	"encoding/xml"
+	"testing"
+)
+
+func TestAutodiscover(t *testing.T) {
+	// Request by Thunderbird.
+	const body = `<?xml version="1.0" encoding="utf-8"?>
+    <Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/requestschema/2006">
+      <Request>
+        <EMailAddress>test@example.org</EMailAddress>
+        <AcceptableResponseSchema>http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a</AcceptableResponseSchema>
+      </Request>
+    </Autodiscover>
+`
+	var req autodiscoverRequest
+	if err := xml.Unmarshal([]byte(body), &req); err != nil {
+		t.Fatalf("unmarshal autodiscover request: %v", err)
+	}
+
+	if req.Request.EmailAddress != "test@example.org" {
+		t.Fatalf("emailaddress: got %q, expected %q", req.Request.EmailAddress, "test@example.org")
+	}
+}

http/mtasts.go

@@ -0,0 +1,64 @@
+package http
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/mtasts"
+)
+
+func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
+	log := xlog.WithCid(mox.Cid())
+
+	if !strings.HasPrefix(r.Host, "mta-sts.") {
+		http.NotFound(w, r)
+		return
+	}
+	domain, err := dns.ParseDomain(strings.TrimPrefix(r.Host, "mta-sts."))
+	if err != nil {
+		log.Errorx("mtasts policy request: bad domain", err, mlog.Field("host", r.Host))
+		http.NotFound(w, r)
+		return
+	}
+
+	conf, _ := mox.Conf.Domain(domain)
+	sts := conf.MTASTS
+	if sts == nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	var mxs []mtasts.STSMX
+	for _, s := range sts.MX {
+		var mx mtasts.STSMX
+		if strings.HasPrefix(s, "*.") {
+			mx.Wildcard = true
+			s = s[2:]
+		}
+		d, err := dns.ParseDomain(s)
+		if err != nil {
+			log.Errorx("bad domain in mtasts config", err, mlog.Field("domain", s))
+			http.Error(w, "500 - internal server error - invalid domain in configuration", http.StatusInternalServerError)
+			return
+		}
+		mx.Domain = d
+		mxs = append(mxs, mx)
+	}
+	if len(mxs) == 0 {
+		mxs = []mtasts.STSMX{{Domain: mox.Conf.Static.HostnameDomain}}
+	}
+
+	policy := mtasts.Policy{
+		Version:       "STSv1",
+		Mode:          sts.Mode,
+		MaxAgeSeconds: int(sts.MaxAge / time.Second),
+		MX:            mxs,
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("Cache-Control", "no-cache, max-age=0")
+	w.Write([]byte(policy.String()))
+}

http/mtasts_test.go

@@ -0,0 +1,3 @@
+package http
+
+// todo: write tests for mtasts handler

http/web.go

@@ -0,0 +1,240 @@
+// Package http provides HTTP listeners/servers, for
+// autoconfiguration/autodiscovery, the account and admin web interface and
+// MTA-STS policies.
+package http
+
+import (
+	"crypto/tls"
+	"fmt"
+	golog "log"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	_ "net/http/pprof"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+)
+
+var xlog = mlog.New("http")
+
+// Set some http headers that should prevent potential abuse. Better safe than sorry.
+func safeHeaders(fn http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("X-Frame-Options", "deny")
+		h.Set("X-Content-Type-Options", "nosniff")
+		h.Set("Content-Security-Policy", "default-src 'self' 'unsafe-inline' data:")
+		h.Set("Referrer-Policy", "same-origin")
+		fn(w, r)
+	}
+}
+
+// ListenAndServe starts listeners for HTTP, including those required for ACME to
+// generate TLS certificates.
+func ListenAndServe() {
+	type serve struct {
+		kinds     []string
+		tlsConfig *tls.Config
+		mux       *http.ServeMux
+	}
+
+	for name, l := range mox.Conf.Static.Listeners {
+		portServe := map[int]serve{}
+
+		var ensureServe func(https bool, port int, kind string) serve
+		ensureServe = func(https bool, port int, kind string) serve {
+			s, ok := portServe[port]
+			if !ok {
+				s = serve{nil, nil, &http.ServeMux{}}
+			}
+			s.kinds = append(s.kinds, kind)
+			if https && port == 443 && l.TLS.ACME != "" {
+				s.tlsConfig = l.TLS.ACMEConfig
+			} else if https {
+				s.tlsConfig = l.TLS.Config
+				if l.TLS.ACME != "" {
+					ensureServe(true, 443, "acme-tls-alpn-01")
+				}
+			}
+			portServe[port] = s
+			return s
+		}
+
+		if l.SMTP.Enabled && !l.SMTP.NoSTARTTLS || l.Submissions.Enabled || l.IMAPS.Enabled {
+			ensureServe(true, 443, "acme-tls-alpn01")
+		}
+
+		if l.AdminHTTP.Enabled {
+			srv := ensureServe(false, config.Port(l.AdminHTTP.Port, 80), "admin-http")
+			srv.mux.HandleFunc("/", safeHeaders(adminIndex))
+			srv.mux.HandleFunc("/admin/", safeHeaders(adminHandle))
+			srv.mux.HandleFunc("/account/", safeHeaders(accountHandle))
+		}
+		if l.AdminHTTPS.Enabled {
+			srv := ensureServe(true, config.Port(l.AdminHTTPS.Port, 443), "admin-https")
+			srv.mux.HandleFunc("/", safeHeaders(adminIndex))
+			srv.mux.HandleFunc("/admin/", safeHeaders(adminHandle))
+			srv.mux.HandleFunc("/account/", safeHeaders(accountHandle))
+		}
+		if l.MetricsHTTP.Enabled {
+			srv := ensureServe(false, config.Port(l.MetricsHTTP.Port, 8010), "metrics-http")
+			srv.mux.Handle("/metrics", safeHeaders(promhttp.Handler().ServeHTTP))
+			srv.mux.HandleFunc("/", safeHeaders(func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.Path != "/" {
+					http.NotFound(w, r)
+					return
+				} else if r.Method != "GET" {
+					http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+					return
+				}
+				w.Header().Set("Content-Type", "text/html")
+				fmt.Fprint(w, `<html><body>see <a href="/metrics">/metrics</a></body></html>`)
+			}))
+		}
+		if l.AutoconfigHTTPS.Enabled {
+			srv := ensureServe(true, 443, "autoconfig-https")
+			srv.mux.HandleFunc("/mail/config-v1.1.xml", safeHeaders(autoconfHandle(l)))
+			srv.mux.HandleFunc("/autodiscover/autodiscover.xml", safeHeaders(autodiscoverHandle(l)))
+		}
+		if l.MTASTSHTTPS.Enabled {
+			srv := ensureServe(true, 443, "mtasts-https")
+			srv.mux.HandleFunc("/.well-known/mta-sts.txt", safeHeaders(mtastsPolicyHandle))
+		}
+		if l.PprofHTTP.Enabled {
+			// Importing net/http/pprof registers handlers on the default serve mux.
+			port := config.Port(l.PprofHTTP.Port, 8011)
+			if _, ok := portServe[port]; ok {
+				xlog.Fatal("cannot serve pprof on same endpoint as other http services")
+			}
+			portServe[port] = serve{[]string{"pprof-http"}, nil, http.DefaultServeMux}
+		}
+
+		// We'll explicitly ensure these TLS certs exist (e.g. are created with ACME)
+		// immediately after startup. We only do so for our explicitly hostnames, not for
+		// autoconfig or mta-sts DNS records, they can be requested on demand (perhaps
+		// never).
+		ensureHosts := map[dns.Domain]struct{}{}
+
+		if l.TLS != nil && l.TLS.ACME != "" {
+			m := mox.Conf.Static.ACME[l.TLS.ACME].Manager
+
+			m.AllowHostname(mox.Conf.Static.HostnameDomain)
+			ensureHosts[mox.Conf.Static.HostnameDomain] = struct{}{}
+			if l.HostnameDomain.ASCII != "" {
+				m.AllowHostname(l.HostnameDomain)
+				ensureHosts[l.HostnameDomain] = struct{}{}
+			}
+
+			go func() {
+				// Just in case someone adds quite some domains to their config. We don't want to
+				// hit any ACME rate limits.
+				if len(ensureHosts) > 10 {
+					return
+				}
+
+				time.Sleep(1 * time.Second)
+				i := 0
+				for hostname := range ensureHosts {
+					if i > 0 {
+						// Sleep just a little. We don't want to hammer our ACME provider, e.g. Let's Encrypt.
+						time.Sleep(10 * time.Second)
+					}
+					i++
+
+					hello := &tls.ClientHelloInfo{
+						ServerName: hostname.ASCII,
+
+						// Make us fetch an ECDSA P256 cert.
+						// We add TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 to get around the ecDSA check in autocert.
+						CipherSuites:      []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_AES_128_GCM_SHA256},
+						SupportedCurves:   []tls.CurveID{tls.CurveP256},
+						SignatureSchemes:  []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256},
+						SupportedVersions: []uint16{tls.VersionTLS13},
+					}
+					xlog.Print("ensuring certificate availability", mlog.Field("hostname", hostname))
+					if _, err := m.Manager.GetCertificate(hello); err != nil {
+						xlog.Errorx("requesting automatic certificate", err, mlog.Field("hostname", hostname))
+					}
+				}
+			}()
+		}
+
+		for port, srv := range portServe {
+			for _, ip := range l.IPs {
+				listenAndServe(ip, port, srv.tlsConfig, name, srv.kinds, srv.mux)
+			}
+		}
+	}
+}
+
+func adminIndex(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
+		return
+	}
+	if r.Method != "GET" {
+		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+		return
+	}
+
+	const html = `<!doctype html>
+<html>
+	<head>
+		<title>mox</title>
+		<meta name="viewport" content="width=device-width" />
+		<style>
+body, html { font-family: ubuntu, lato, sans-serif; font-size: 16px; padding: 1em; }
+* { margin: 0; padding: 0; box-sizing: border-box; }
+h1, h2, h3, h4 { margin-bottom: 1ex; }
+h1 { font-size: 1.2rem; }
+		</style>
+	</head>
+	<body>
+		<h1>mox</h1>
+		<div><a href="/account/">/account/</a>, for regular login</div>
+		<div><a href="/admin/">/admin/</a>, for adminstrators</div>
+	</body>
+</html>`
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Write([]byte(html))
+}
+
+func listenAndServe(ip string, port int, tlsConfig *tls.Config, name string, kinds []string, mux *http.ServeMux) {
+	addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
+
+	var protocol string
+	var ln net.Listener
+	var err error
+	if tlsConfig == nil {
+		protocol = "http"
+		xlog.Print("http listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr))
+		ln, err = net.Listen(mox.Network(ip), addr)
+		if err != nil {
+			xlog.Fatalx("http: listen"+mox.LinuxSetcapHint(err), err, mlog.Field("addr", addr))
+		}
+	} else {
+		protocol = "https"
+		xlog.Print("https listener", mlog.Field("name", name), mlog.Field("kinds", strings.Join(kinds, ",")), mlog.Field("address", addr))
+		ln, err = tls.Listen(mox.Network(ip), addr, tlsConfig)
+		if err != nil {
+			xlog.Fatalx("https: listen"+mox.LinuxSetcapHint(err), err, mlog.Field("addr", addr))
+		}
+	}
+
+	server := &http.Server{
+		Handler:   mux,
+		TLSConfig: tlsConfig,
+		ErrorLog:  golog.New(mlog.ErrWriter(xlog.Fields(mlog.Field("pkg", "net/http")), mlog.LevelInfo, protocol+" error"), "", 0),
+	}
+	go func() {
+		err := server.Serve(ln)
+		xlog.Fatalx(protocol+": serve", err)
+	}()
+}

imapclient/client.go

@@ -0,0 +1,293 @@
+/*
+Package imapclient provides an IMAP4 client, primarily for testing the IMAP4 server.
+
+Commands can be sent to the server free-form, but responses are parsed strictly.
+Behaviour that may not be required by the IMAP4 specification may be expected by
+this client.
+*/
+package imapclient
+
+/*
+- Try to keep the parsing method names and the types similar to the ABNF names in the RFCs.
+
+- todo: have mode for imap4rev1 vs imap4rev2, refusing what is not allowed. we are accepting too much now.
+- todo: stricter parsing. xnonspace() and xword() should be replaced by proper parsers.
+*/
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"reflect"
+	"strings"
+)
+
+// Conn is an IMAP connection to a server.
+type Conn struct {
+	conn      net.Conn
+	r         *bufio.Reader
+	panic     bool
+	tagGen    int
+	record    bool // If true, bytes read are added to recordBuf. recorded() resets.
+	recordBuf []byte
+
+	LastTag      string
+	CapAvailable map[Capability]struct{} // Capabilities available at server, from CAPABILITY command or response code.
+	CapEnabled   map[Capability]struct{} // Capabilities enabled through ENABLE command.
+}
+
+// Error is a parse or other protocol error.
+type Error struct{ err error }
+
+func (e Error) Error() string {
+	return e.err.Error()
+}
+
+func (e Error) Unwrap() error {
+	return e.err
+}
+
+// New creates a new client on conn.
+//
+// If xpanic is true, functions that would return an error instead panic. For parse
+// errors, the resulting stack traces show typically show what was being parsed.
+//
+// The initial untagged greeting response is read and must be "OK".
+func New(conn net.Conn, xpanic bool) (client *Conn, rerr error) {
+	c := Conn{
+		conn:         conn,
+		r:            bufio.NewReader(conn),
+		panic:        xpanic,
+		CapAvailable: map[Capability]struct{}{},
+		CapEnabled:   map[Capability]struct{}{},
+	}
+
+	defer c.recover(&rerr)
+	tag := c.xnonspace()
+	if tag != "*" {
+		c.xerrorf("expected untagged *, got %q", tag)
+	}
+	c.xspace()
+	ut := c.xuntagged()
+	switch x := ut.(type) {
+	case UntaggedResult:
+		if x.Status != OK {
+			c.xerrorf("greeting, got status %q, expected OK", x.Status)
+		}
+		return &c, nil
+	case UntaggedPreauth:
+		c.xerrorf("greeting: unexpected preauth")
+	case UntaggedBye:
+		c.xerrorf("greeting: server sent bye")
+	default:
+		c.xerrorf("unexpected untagged %v", ut)
+	}
+	panic("not reached")
+}
+
+func (c *Conn) recover(rerr *error) {
+	if c.panic {
+		return
+	}
+
+	x := recover()
+	if x == nil {
+		return
+	}
+	err, ok := x.(Error)
+	if !ok {
+		panic(x)
+	}
+	*rerr = err
+}
+
+func (c *Conn) xerrorf(format string, args ...any) {
+	panic(Error{fmt.Errorf(format, args...)})
+}
+
+func (c *Conn) xcheckf(err error, format string, args ...any) {
+	if err != nil {
+		c.xerrorf("%s: %w", fmt.Sprintf(format, args...), err)
+	}
+}
+
+func (c *Conn) xcheck(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// Commandf writes a free-form IMAP command to the server.
+// If tag is empty, a next unique tag is assigned.
+func (c *Conn) Commandf(tag string, format string, args ...any) (rerr error) {
+	defer c.recover(&rerr)
+
+	if tag == "" {
+		tag = c.nextTag()
+	}
+	c.LastTag = tag
+
+	_, err := fmt.Fprintf(c.conn, "%s %s\r\n", tag, fmt.Sprintf(format, args...))
+	c.xcheckf(err, "write command")
+	return
+}
+
+func (c *Conn) nextTag() string {
+	c.tagGen++
+	return fmt.Sprintf("x%03d", c.tagGen)
+}
+
+// Response reads from the IMAP server until a tagged response line is found.
+// The tag must be the same as the tag for the last written command.
+// Result holds the status of the command. The caller must check if this the status is OK.
+func (c *Conn) Response() (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+
+	for {
+		tag := c.xnonspace()
+		c.xspace()
+		if tag == "*" {
+			untagged = append(untagged, c.xuntagged())
+			continue
+		}
+
+		if tag != c.LastTag {
+			c.xerrorf("got tag %q, expected %q", tag, c.LastTag)
+		}
+
+		status := c.xstatus()
+		c.xspace()
+		result = c.xresult(status)
+		c.xcrlf()
+		return
+	}
+}
+
+// ReadUntagged reads a single untagged response line.
+// Useful for reading lines from IDLE.
+func (c *Conn) ReadUntagged() (untagged Untagged, rerr error) {
+	defer c.recover(&rerr)
+
+	tag := c.xnonspace()
+	if tag != "*" {
+		c.xerrorf("got tag %q, expected untagged", tag)
+	}
+	c.xspace()
+	ut := c.xuntagged()
+	return ut, nil
+}
+
+// Readline reads a line, including CRLF.
+// Used with IDLE and synchronous literals.
+func (c *Conn) Readline() (line string, rerr error) {
+	defer c.recover(&rerr)
+
+	line, err := c.r.ReadString('\n')
+	c.xcheckf(err, "read line")
+	return line, nil
+}
+
+// ReadContinuation reads a line. If it is a continuation, i.e. starts with a +, it
+// is returned without leading "+ " and without trailing crlf. Otherwise, a command
+// response is returned. A successfully read continuation can return an empty line.
+// Callers should check rerr and result.Status being empty to check if a
+// continuation was read.
+func (c *Conn) ReadContinuation() (line string, untagged []Untagged, result Result, rerr error) {
+	if !c.peek('+') {
+		untagged, result, rerr = c.Response()
+		c.xcheckf(rerr, "reading non-continuation response")
+		c.xerrorf("response status %q, expected OK", result.Status)
+	}
+	c.xtake("+ ")
+	line, err := c.Readline()
+	c.xcheckf(err, "read line")
+	line = strings.TrimSuffix(line, "\r\n")
+	return
+}
+
+// Writelinef writes the formatted format and args as a single line, adding CRLF.
+// Used with IDLE and synchronous literals.
+func (c *Conn) Writelinef(format string, args ...any) (rerr error) {
+	defer c.recover(&rerr)
+
+	s := fmt.Sprintf(format, args...)
+	_, err := fmt.Fprintf(c.conn, "%s\r\n", s)
+	c.xcheckf(err, "writeline")
+	return nil
+}
+
+// Write writes directly to the connection. Write errors do take the connections
+// panic mode into account, i.e. Write can panic.
+func (c *Conn) Write(buf []byte) (n int, rerr error) {
+	defer c.recover(&rerr)
+
+	n, rerr = c.conn.Write(buf)
+	c.xcheckf(rerr, "write")
+	return n, nil
+}
+
+// WriteSyncLiteral first writes the synchronous literal size, then read the
+// continuation "+" and finally writes the data.
+func (c *Conn) WriteSyncLiteral(s string) (rerr error) {
+	defer c.recover(&rerr)
+
+	_, err := fmt.Fprintf(c.conn, "{%d}\r\n", len(s))
+	c.xcheckf(err, "write sync literal size")
+	line, err := c.Readline()
+	c.xcheckf(err, "read line")
+	if !strings.HasPrefix(line, "+") {
+		c.xerrorf("no continuation received for sync literal")
+	}
+	_, err = c.conn.Write([]byte(s))
+	c.xcheckf(err, "write literal data")
+	return nil
+}
+
+// Transactf writes format and args as an IMAP command, using Commandf with an
+// empty tag. I.e. format must not contain a tag. Transactf then reads a response
+// using ReadResponse and checks the result status is OK.
+func (c *Conn) Transactf(format string, args ...any) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+
+	err := c.Commandf("", format, args...)
+	if err != nil {
+		return nil, Result{}, err
+	}
+	return c.ResponseOK()
+}
+
+func (c *Conn) ResponseOK() (untagged []Untagged, result Result, rerr error) {
+	untagged, result, rerr = c.Response()
+	if rerr != nil {
+		return nil, Result{}, rerr
+	}
+	if result.Status != OK {
+		c.xerrorf("response status %q, expected OK", result.Status)
+	}
+	return untagged, result, rerr
+}
+
+func (c *Conn) xgetUntagged(l []Untagged, dst any) {
+	if len(l) != 1 {
+		c.xerrorf("got %d untagged, expected 1: %v", len(l), l)
+	}
+	got := l[0]
+	gotv := reflect.ValueOf(got)
+	dstv := reflect.ValueOf(dst)
+	if gotv.Type() != dstv.Type().Elem() {
+		c.xerrorf("got %v, expected %v", gotv.Type(), dstv.Type().Elem())
+	}
+	dstv.Elem().Set(gotv)
+}
+
+// Close closes the connection without writing anything to the server.
+// You may want to call Logout. Closing a connection with a mailbox with deleted
+// message not yet expunged will not expunge those messages.
+func (c *Conn) Close() error {
+	var err error
+	if c.conn != nil {
+		err = c.conn.Close()
+		c.conn = nil
+	}
+	return err
+}

imapclient/cmds.go

@@ -0,0 +1,292 @@
+package imapclient
+
+import (
+	"bufio"
+	"crypto/tls"
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/scram"
+)
+
+// Capability requests a list of capabilities from the server. They are returned in
+// an UntaggedCapability response. The server also sends capabilities in initial
+// server greeting, in the response code.
+func (c *Conn) Capability() (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("capability")
+}
+
+// Noop does nothing on its own, but a server will return any pending untagged
+// responses for new message delivery and changes to mailboxes.
+func (c *Conn) Noop() (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("capability")
+}
+
+// Logout ends the IMAP session by writing a LOGOUT command. Close must still be
+// closed on this client to close the socket.
+func (c *Conn) Logout() (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("logout")
+}
+
+// Starttls enables TLS on the connection with the STARTTLS command.
+func (c *Conn) Starttls(config *tls.Config) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	untagged, result, rerr = c.Transactf("starttls")
+	c.xcheckf(rerr, "starttls command")
+	conn := tls.Client(c.conn, config)
+	err := conn.Handshake()
+	c.xcheckf(err, "tls handshake")
+	c.conn = conn
+	c.r = bufio.NewReader(conn)
+	return untagged, result, nil
+}
+
+// Login authenticates with username and password
+func (c *Conn) Login(username, password string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("login %s %s", astring(username), astring(password))
+}
+
+// Authenticate with plaintext password using AUTHENTICATE PLAIN.
+func (c *Conn) AuthenticatePlain(username, password string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+
+	untagged, result, rerr = c.Transactf("authenticate plain %s", base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "\u0000%s\u0000%s", username, password)))
+	return
+}
+
+// Authenticate with SCRAM-SHA-256, where the password is not exchanged in original
+// plaintext form, but only derived hashes are exchanged by both parties as proof
+// of knowledge of password.
+func (c *Conn) AuthenticateSCRAMSHA256(username, password string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+
+	sc := scram.NewClient(username, "")
+	clientFirst, err := sc.ClientFirst()
+	c.xcheckf(err, "scram clientFirst")
+	c.LastTag = c.nextTag()
+	err = c.Writelinef("%s authenticate scram-sha-256 %s", c.LastTag, base64.StdEncoding.EncodeToString([]byte(clientFirst)))
+	c.xcheckf(err, "writing command line")
+
+	xreadContinuation := func() []byte {
+		var line string
+		line, untagged, result, rerr = c.ReadContinuation()
+		c.xcheckf(err, "read continuation")
+		if result.Status != "" {
+			c.xerrorf("unexpected status %q", result.Status)
+		}
+		buf, err := base64.StdEncoding.DecodeString(line)
+		c.xcheckf(err, "parsing base64 from remote")
+		return buf
+	}
+
+	serverFirst := xreadContinuation()
+	clientFinal, err := sc.ServerFirst(serverFirst, password)
+	c.xcheckf(err, "scram clientFinal")
+	err = c.Writelinef("%s", base64.StdEncoding.EncodeToString([]byte(clientFinal)))
+	c.xcheckf(err, "write scram clientFinal")
+
+	serverFinal := xreadContinuation()
+	err = sc.ServerFinal(serverFinal)
+	c.xcheckf(err, "scram serverFinal")
+
+	// We must send a response to the server continuation line, but we have nothing to say. ../rfc/9051:6221
+	err = c.Writelinef("%s", base64.StdEncoding.EncodeToString(nil))
+	c.xcheckf(err, "scram client end")
+
+	return c.ResponseOK()
+}
+
+// Enable enables capabilities for use with the connection, verifying the server has indeed enabled them.
+func (c *Conn) Enable(capabilities ...string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+
+	untagged, result, rerr = c.Transactf("enable %s", strings.Join(capabilities, " "))
+	c.xcheck(rerr)
+	var enabled UntaggedEnabled
+	c.xgetUntagged(untagged, &enabled)
+	got := map[string]struct{}{}
+	for _, cap := range enabled {
+		got[cap] = struct{}{}
+	}
+	for _, cap := range capabilities {
+		if _, ok := got[cap]; !ok {
+			c.xerrorf("capability %q not enabled by server", cap)
+		}
+	}
+	return
+}
+
+// Select opens mailbox as active mailbox.
+func (c *Conn) Select(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("select %s", astring(mailbox))
+}
+
+// Examine opens mailbox as active mailbox read-only.
+func (c *Conn) Examine(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("examine %s", astring(mailbox))
+}
+
+// Create makes a new mailbox on the server.
+func (c *Conn) Create(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("create %s", astring(mailbox))
+}
+
+// Delete removes an entire mailbox and its messages.
+func (c *Conn) Delete(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("delete %s", astring(mailbox))
+}
+
+// Rename changes the name of a mailbox and all its child mailboxes.
+func (c *Conn) Rename(omailbox, nmailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("rename %s %s", astring(omailbox), astring(nmailbox))
+}
+
+// Subscribe marks a mailbox as subscribed. The mailbox does not have to exist. It
+// is not an error if the mailbox is already subscribed.
+func (c *Conn) Subscribe(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("subscribe %s", astring(mailbox))
+}
+
+// Unsubscribe marks a mailbox as unsubscribed.
+func (c *Conn) Unsubscribe(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("unsubscribe %s", astring(mailbox))
+}
+
+// List lists mailboxes with the basic LIST syntax.
+// Pattern can contain * (match any) or % (match any except hierarchy delimiter).
+func (c *Conn) List(pattern string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf(`list "" %s`, astring(pattern))
+}
+
+// ListFull lists mailboxes with the extended LIST syntax requesting all supported data.
+// Pattern can contain * (match any) or % (match any except hierarchy delimiter).
+func (c *Conn) ListFull(subscribedOnly bool, patterns ...string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	var subscribedStr string
+	if subscribedOnly {
+		subscribedStr = "subscribed recursivematch"
+	}
+	for i, s := range patterns {
+		patterns[i] = astring(s)
+	}
+	return c.Transactf(`list (%s) "" (%s) return (subscribed children special-use status (messages uidnext uidvalidity unseen deleted size recent appendlimit))`, subscribedStr, strings.Join(patterns, " "))
+}
+
+// Namespace returns the hiearchy separator in an UntaggedNamespace response with personal/shared/other namespaces if present.
+func (c *Conn) Namespace() (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("namespace")
+}
+
+// Status requests information about a mailbox, such as number of messages, size, etc.
+func (c *Conn) Status(mailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("status %s", astring(mailbox))
+}
+
+// Append adds message to mailbox with flags and optional receive time.
+func (c *Conn) Append(mailbox string, flags []string, received *time.Time, message []byte) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	var date string
+	if received != nil {
+		date = ` "` + received.Format("_2-Jan-2006 15:04:05 -0700") + `"`
+	}
+	return c.Transactf("append %s (%s)%s {%d+}\r\n%s", astring(mailbox), strings.Join(flags, " "), date, len(message), message)
+}
+
+// note: No idle command. Idle is better implemented by writing the request and reading and handling the responses as they come in.
+
+// CloseMailbox closes the currently selected/active mailbox, permanently removing
+// any messages marked with \Deleted.
+func (c *Conn) CloseMailbox() (untagged []Untagged, result Result, rerr error) {
+	return c.Transactf("close")
+}
+
+// Unselect closes the currently selected/active mailbox, but unlike CloseMailbox
+// does not permanently remove any messages marked with \Deleted.
+func (c *Conn) Unselect() (untagged []Untagged, result Result, rerr error) {
+	return c.Transactf("unselect")
+}
+
+// Expunge removes messages marked as deleted for the selected mailbox.
+func (c *Conn) Expunge() (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("expunge")
+}
+
+// UIDExpunge is like expunge, but only removes messages matched uidSet.
+func (c *Conn) UIDExpunge(uidSet NumSet) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("uid expunge %s", uidSet.String())
+}
+
+// Note: No search, fetch command yet due to its large syntax.
+
+// StoreFlagsSet stores a new set of flags for messages from seqset with the STORE command.
+// If silent, no untagged responses with the updated flags will be sent by the server.
+func (c *Conn) StoreFlagsSet(seqset string, silent bool, flags ...string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	item := "flags"
+	if silent {
+		item += ".silent"
+	}
+	return c.Transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
+}
+
+// StoreFlagsAdd is like StoreFlagsSet, but only adds flags, leaving current flags on the message intact.
+func (c *Conn) StoreFlagsAdd(seqset string, silent bool, flags ...string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	item := "+flags"
+	if silent {
+		item += ".silent"
+	}
+	return c.Transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
+}
+
+// StoreFlagsClear is like StoreFlagsSet, but only removes flags, leaving other flags on the message intact.
+func (c *Conn) StoreFlagsClear(seqset string, silent bool, flags ...string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	item := "-flags"
+	if silent {
+		item += ".silent"
+	}
+	return c.Transactf("store %s %s (%s)", seqset, item, strings.Join(flags, " "))
+}
+
+// Copy adds messages from the sequences in seqSet in the currently selected/active mailbox to dstMailbox.
+func (c *Conn) Copy(seqSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("copy %s %s", seqSet.String(), astring(dstMailbox))
+}
+
+// UIDCopy is like copy, but operates on UIDs.
+func (c *Conn) UIDCopy(uidSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("uid copy %s %s", uidSet.String(), astring(dstMailbox))
+}
+
+// Move moves messages from the sequences in seqSet in the currently selected/active mailbox to dstMailbox.
+func (c *Conn) Move(seqSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("move %s %s", seqSet.String(), astring(dstMailbox))
+}
+
+// UIDMove is like move, but operates on UIDs.
+func (c *Conn) UIDMove(uidSet NumSet, dstMailbox string) (untagged []Untagged, result Result, rerr error) {
+	defer c.recover(&rerr)
+	return c.Transactf("uid move %s %s", uidSet.String(), astring(dstMailbox))
+}

imapclient/protocol.go

@@ -0,0 +1,452 @@
+package imapclient
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Capability is a known string for with the ENABLED and CAPABILITY command.
+type Capability string
+
+const (
+	CapIMAP4rev1     Capability = "IMAP4rev1"
+	CapIMAP4rev2     Capability = "IMAP4rev2"
+	CapLoginDisabled Capability = "LOGINDISABLED"
+	CapStarttls      Capability = "STARTTLS"
+	CapAuthPlain     Capability = "AUTH=PLAIN"
+	CapLiteralPlus   Capability = "LITERAL+"
+	CapLiteralMinus  Capability = "LITERAL-"
+	CapIdle          Capability = "IDLE"
+	CapNamespace     Capability = "NAMESPACE"
+	CapBinary        Capability = "BINARY"
+	CapUnselect      Capability = "UNSELECT"
+	CapUidplus       Capability = "UIDPLUS"
+	CapEsearch       Capability = "ESEARCH"
+	CapEnable        Capability = "ENABLE"
+	CapSave          Capability = "SAVE"
+	CapListExtended  Capability = "LIST-EXTENDED"
+	CapSpecialUse    Capability = "SPECIAL-USE"
+	CapMove          Capability = "MOVE"
+	CapUTF8Only      Capability = "UTF8=ONLY"
+	CapUTF8Accept    Capability = "UTF8=ACCEPT"
+	CapID            Capability = "ID" // ../rfc/2971:80
+)
+
+// Status is the tagged final result of a command.
+type Status string
+
+const (
+	BAD Status = "BAD" // Syntax error.
+	NO  Status = "NO"  // Command failed.
+	OK  Status = "OK"  // Command succeeded.
+)
+
+// Result is the final response for a command, indicating success or failure.
+type Result struct {
+	Status Status
+	RespText
+}
+
+// CodeArg represents a response code with arguments, i.e. the data between [] in the response line.
+type CodeArg interface {
+	CodeString() string
+}
+
+// CodeOther is a valid but unrecognized response code.
+type CodeOther struct {
+	Code string
+	Args []string
+}
+
+func (c CodeOther) CodeString() string {
+	return c.Code + " " + strings.Join(c.Args, " ")
+}
+
+// CodeWords is a code with space-separated string parameters. E.g. CAPABILITY.
+type CodeWords struct {
+	Code string
+	Args []string
+}
+
+func (c CodeWords) CodeString() string {
+	s := c.Code
+	for _, w := range c.Args {
+		s += " " + w
+	}
+	return s
+}
+
+// CodeList is a code with a list with space-separated strings as parameters. E.g. BADCHARSET, PERMANENTFLAGS.
+type CodeList struct {
+	Code string
+	Args []string // If nil, no list was present. List can also be empty.
+}
+
+func (c CodeList) CodeString() string {
+	s := c.Code
+	if c.Args == nil {
+		return s
+	}
+	return s + "(" + strings.Join(c.Args, " ") + ")"
+}
+
+// CodeUint is a code with a uint32 parameter, e.g. UIDNEXT and UIDVALIDITY.
+type CodeUint struct {
+	Code string
+	Num  uint32
+}
+
+func (c CodeUint) CodeString() string {
+	return fmt.Sprintf("%s %d", c.Code, c.Num)
+}
+
+// "APPENDUID" response code.
+type CodeAppendUID struct {
+	UIDValidity uint32
+	UID         uint32
+}
+
+func (c CodeAppendUID) CodeString() string {
+	return fmt.Sprintf("APPENDUID %d %d", c.UIDValidity, c.UID)
+}
+
+// "COPYUID" response code.
+type CodeCopyUID struct {
+	DestUIDValidity uint32
+	From            []NumRange
+	To              []NumRange
+}
+
+func (c CodeCopyUID) CodeString() string {
+	str := func(l []NumRange) string {
+		s := ""
+		for i, e := range l {
+			if i > 0 {
+				s += ","
+			}
+			s += fmt.Sprintf("%d", e.First)
+			if e.Last != nil {
+				s += fmt.Sprintf(":%d", *e.Last)
+			}
+		}
+		return s
+	}
+	return fmt.Sprintf("COPYUID %d %s %s", c.DestUIDValidity, str(c.From), str(c.To))
+}
+
+// RespText represents a response line minus the leading tag.
+type RespText struct {
+	Code    string  // The first word between [] after the status.
+	CodeArg CodeArg // Set if code has a parameter.
+	More    string  // Any remaining text.
+}
+
+// atom or string.
+func astring(s string) string {
+	if len(s) == 0 {
+		return stringx(s)
+	}
+	for _, c := range s {
+		if c <= ' ' || c >= 0x7f || c == '(' || c == ')' || c == '{' || c == '%' || c == '*' || c == '"' || c == '\\' {
+			stringx(s)
+		}
+	}
+	return s
+}
+
+// imap "string", i.e. double-quoted string or syncliteral.
+func stringx(s string) string {
+	r := `"`
+	for _, c := range s {
+		if c == '\x00' || c == '\r' || c == '\n' {
+			return syncliteral(s)
+		}
+		if c == '\\' || c == '"' {
+			r += `\`
+		}
+		r += string(c)
+	}
+	r += `"`
+	return r
+}
+
+// sync literal, i.e. {<num>}\r\n<num bytes>.
+func syncliteral(s string) string {
+	return fmt.Sprintf("{%d}\r\n", len(s)) + s
+}
+
+// Untagged is a parsed untagged response. See types starting with Untagged.
+// todo: make an interface that the untagged responses implement?
+type Untagged any
+
+type UntaggedBye RespText
+type UntaggedPreauth RespText
+type UntaggedExpunge uint32
+type UntaggedExists uint32
+type UntaggedRecent uint32
+type UntaggedCapability []string
+type UntaggedEnabled []string
+type UntaggedResult Result
+type UntaggedFlags []string
+type UntaggedList struct {
+	// ../rfc/9051:6690
+	Flags     []string
+	Separator byte // 0 for NIL
+	Mailbox   string
+	Extended  []MboxListExtendedItem
+	OldName   string // If present, taken out of Extended.
+}
+type UntaggedFetch struct {
+	Seq   uint32
+	Attrs []FetchAttr
+}
+type UntaggedSearch []uint32
+type UntaggedStatus struct {
+	Mailbox string
+	Attrs   map[string]int64 // Upper case status attributes. ../rfc/9051:7059
+}
+type UntaggedNamespace struct {
+	Personal, Other, Shared []NamespaceDescr
+}
+type UntaggedLsub struct {
+	// ../rfc/3501:4833
+	Flags     []string
+	Separator byte
+	Mailbox   string
+}
+
+// Fields are optional and zero if absent.
+type UntaggedEsearch struct {
+	// ../rfc/9051:6546
+	Correlator string
+	UID        bool
+	Min        uint32
+	Max        uint32
+	All        NumSet
+	Count      *uint32
+	Exts       []EsearchDataExt
+}
+
+// ../rfc/2971:184
+
+type UntaggedID map[string]string
+
+// Extended data in an ESEARCH response.
+type EsearchDataExt struct {
+	Tag   string
+	Value TaggedExtVal
+}
+
+type NamespaceDescr struct {
+	// ../rfc/9051:6769
+	Prefix    string
+	Separator byte // If 0 then separator was absent.
+	Exts      []NamespaceExtension
+}
+
+type NamespaceExtension struct {
+	// ../rfc/9051:6773
+	Key    string
+	Values []string
+}
+
+// FetchAttr represents a FETCH response attribute.
+type FetchAttr interface {
+	Attr() string // Name of attribute.
+}
+
+type NumSet struct {
+	SearchResult bool // True if "$", in which case Ranges is irrelevant.
+	Ranges       []NumRange
+}
+
+func (ns NumSet) IsZero() bool {
+	return !ns.SearchResult && ns.Ranges == nil
+}
+
+func (ns NumSet) String() string {
+	if ns.SearchResult {
+		return "$"
+	}
+	var r string
+	for i, x := range ns.Ranges {
+		if i > 0 {
+			r += ","
+		}
+		r += x.String()
+	}
+	return r
+}
+
+// NumRange is a single number or range.
+type NumRange struct {
+	First uint32  // 0 for "*".
+	Last  *uint32 // Nil if absent, 0 for "*".
+}
+
+func (nr NumRange) String() string {
+	var r string
+	if nr.First == 0 {
+		r += "*"
+	} else {
+		r += fmt.Sprintf("%d", nr.First)
+	}
+	if nr.Last == nil {
+		return r
+	}
+	r += ":"
+	v := *nr.Last
+	if v == 0 {
+		r += "*"
+	} else {
+		r += fmt.Sprintf("%d", v)
+	}
+	return r
+}
+
+type TaggedExtComp struct {
+	String string
+	Comps  []TaggedExtComp // Used for both space-separated and ().
+}
+
+type TaggedExtVal struct {
+	// ../rfc/9051:7111
+	Number *int64
+	SeqSet *NumSet
+	Comp   *TaggedExtComp // If SimpleNumber and SimpleSeqSet is nil, this is a Comp. But Comp is optional and can also be nil. Not great.
+}
+
+type MboxListExtendedItem struct {
+	// ../rfc/9051:6699
+	Tag string
+	Val TaggedExtVal
+}
+
+// "FLAGS" fetch response.
+type FetchFlags []string
+
+func (f FetchFlags) Attr() string { return "FLAGS" }
+
+// "ENVELOPE" fetch response.
+type FetchEnvelope Envelope
+
+func (f FetchEnvelope) Attr() string { return "ENVELOPE" }
+
+// Envelope holds the basic email message fields.
+type Envelope struct {
+	Date                               string
+	Subject                            string
+	From, Sender, ReplyTo, To, CC, BCC []Address
+	InReplyTo, MessageID               string
+}
+
+// Address is an address field in an email message, e.g. To.
+type Address struct {
+	Name, Adl, Mailbox, Host string
+}
+
+// "INTERNALDATE" fetch response.
+type FetchInternalDate string            // todo: parsed time
+func (f FetchInternalDate) Attr() string { return "INTERNALDATE" }
+
+// "RFC822.SIZE" fetch response.
+type FetchRFC822Size int64
+
+func (f FetchRFC822Size) Attr() string { return "RFC822.SIZE" }
+
+// "RFC822" fetch response.
+type FetchRFC822 string
+
+func (f FetchRFC822) Attr() string { return "RFC822" }
+
+// "RFC822.HEADER" fetch response.
+type FetchRFC822Header string
+
+func (f FetchRFC822Header) Attr() string { return "RFC822.HEADER" }
+
+// "RFC82.TEXT" fetch response.
+type FetchRFC822Text string
+
+func (f FetchRFC822Text) Attr() string { return "RFC822.TEXT" }
+
+// "BODYSTRUCTURE" fetch response.
+type FetchBodystructure struct {
+	// ../rfc/9051:6355
+	RespAttr string
+	Body     any // BodyType*
+}
+
+func (f FetchBodystructure) Attr() string { return f.RespAttr }
+
+// "BODY" fetch response.
+type FetchBody struct {
+	// ../rfc/9051:6756 ../rfc/9051:6985
+	RespAttr string
+	Section  string // todo: parse more ../rfc/9051:6985
+	Offset   int32
+	Body     string
+}
+
+func (f FetchBody) Attr() string { return f.RespAttr }
+
+// BodyFields is part of a FETCH BODY[] response.
+type BodyFields struct {
+	Params                       [][2]string
+	ContentID, ContentDescr, CTE string
+	Octets                       int32
+}
+
+// BodyTypeMpart represents the body structure a multipart message, with subparts and the multipart media subtype. Used in a FETCH response.
+type BodyTypeMpart struct {
+	// ../rfc/9051:6411
+	Bodies       []any // BodyTypeBasic, BodyTypeMsg, BodyTypeText
+	MediaSubtype string
+}
+
+// BodyTypeBasic represents basic information about a part, used in a FETCH response.
+type BodyTypeBasic struct {
+	// ../rfc/9051:6407
+	MediaType, MediaSubtype string
+	BodyFields              BodyFields
+}
+
+// BodyTypeMsg represents an email message as a body structure, used in a FETCH response.
+type BodyTypeMsg struct {
+	// ../rfc/9051:6415
+	MediaType, MediaSubtype string
+	BodyFields              BodyFields
+	Envelope                Envelope
+	Bodystructure           any // One of the BodyType*
+	Lines                   int64
+}
+
+// BodyTypeText represents a text part as a body structure, used in a FETCH response.
+type BodyTypeText struct {
+	// ../rfc/9051:6418
+	MediaType, MediaSubtype string
+	BodyFields              BodyFields
+	Lines                   int64
+}
+
+// "BINARY" fetch response.
+type FetchBinary struct {
+	RespAttr string
+	Parts    []uint32 // Can be nil.
+	Data     string
+}
+
+func (f FetchBinary) Attr() string { return f.RespAttr }
+
+// "BINARY.SIZE" fetch response.
+type FetchBinarySize struct {
+	RespAttr string
+	Parts    []uint32
+	Size     int64
+}
+
+func (f FetchBinarySize) Attr() string { return f.RespAttr }
+
+// "UID" fetch response.
+type FetchUID uint32
+
+func (f FetchUID) Attr() string { return "UID" }

imapserver/append_test.go

@@ -0,0 +1,77 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestAppend(t *testing.T) {
+	defer mockUIDValidity()()
+
+	tc := start(t) // note: with switchboard because this connection stays alive unlike tc2.
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t) // note: without switchboard because this connection will break during tests.
+	defer tc2.close()
+
+	tc3 := startNoSwitchboard(t)
+	defer tc3.close()
+
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("inbox")
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+	tc3.client.Login("mjl@mox.example", "testtest")
+
+	tc2.transactf("bad", "append")              // Missing params.
+	tc2.transactf("bad", `append inbox`)        // Missing message.
+	tc2.transactf("bad", `append inbox "test"`) // Message must be literal.
+
+	// Syntax error for line ending in literal causes connection abort.
+	tc2.transactf("bad", "append inbox (\\Badflag) {1+}\r\nx") // Unknown flag.
+	tc2 = startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("inbox")
+
+	tc2.transactf("bad", "append inbox () \"bad time\" {1+}\r\nx") // Bad time.
+	tc2 = startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("inbox")
+
+	tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}")
+	tc2.xcode("TRYCREATE")
+
+	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc2.xuntagged(imapclient.UntaggedExists(1))
+	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 1})
+
+	tc.transactf("ok", "noop")
+	uid1 := imapclient.FetchUID(1)
+	flagsSeen := imapclient.FetchFlags{`\Seen`}
+	tc.xuntagged(imapclient.UntaggedExists(1), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, flagsSeen}})
+	tc3.transactf("ok", "noop")
+	tc3.xuntagged() // Inbox is not selected, nothing to report.
+
+	tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" UTF8 ({34+}\r\ncontent-type: text/plain;;\r\n\r\ntest)")
+	tc2.xuntagged(imapclient.UntaggedExists(2))
+	tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 2})
+
+	// Messages that we cannot parse are marked as application/octet-stream. Perhaps
+	// the imap client knows how to deal with them.
+	tc2.transactf("ok", "uid fetch 2 body")
+	uid2 := imapclient.FetchUID(2)
+	xbs := imapclient.FetchBodystructure{
+		RespAttr: "BODY",
+		Body: imapclient.BodyTypeBasic{
+			MediaType:    "APPLICATION",
+			MediaSubtype: "OCTET-STREAM",
+			BodyFields: imapclient.BodyFields{
+				Octets: 4,
+			},
+		},
+	}
+	tc2.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, xbs}})
+}

imapserver/authenticate_test.go

@@ -0,0 +1,110 @@
+package imapserver
+
+import (
+	"encoding/base64"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/mjl-/mox/scram"
+)
+
+func TestAuthenticatePlain(t *testing.T) {
+	tc := start(t)
+
+	tc.transactf("no", "authenticate bogus ")
+	tc.transactf("bad", "authenticate plain not base64...")
+	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000baduser\u0000badpass")))
+	tc.xcode("AUTHENTICATIONFAILED")
+	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000badpass")))
+	tc.xcode("AUTHENTICATIONFAILED")
+	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl\u0000badpass"))) // Need email, not account.
+	tc.xcode("AUTHENTICATIONFAILED")
+	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000test")))
+	tc.xcode("AUTHENTICATIONFAILED")
+	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000testtesttest")))
+	tc.xcode("AUTHENTICATIONFAILED")
+	tc.transactf("bad", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000")))
+	tc.xcode("")
+	tc.transactf("no", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("other\u0000mjl@mox.example\u0000testtest")))
+	tc.xcode("AUTHORIZATIONFAILED")
+	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000testtest")))
+	tc.close()
+
+	tc = start(t)
+	tc.transactf("ok", "authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("mjl@mox.example\u0000mjl@mox.example\u0000testtest")))
+	tc.close()
+
+	tc = start(t)
+	tc.client.AuthenticatePlain("mjl@mox.example", "testtest")
+	tc.close()
+
+	tc = start(t)
+	defer tc.close()
+
+	tc.cmdf("", "authenticate plain")
+	tc.readprefixline("+ ")
+	tc.writelinef("*") // Aborts.
+	tc.readstatus("bad")
+
+	tc.cmdf("", "authenticate plain")
+	tc.readprefixline("+")
+	tc.writelinef("%s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000testtest")))
+	tc.readstatus("ok")
+}
+
+func TestAuthenticateSCRAMSHA256(t *testing.T) {
+	tc := start(t)
+	tc.client.AuthenticateSCRAMSHA256("mjl@mox.example", "testtest")
+	tc.close()
+
+	auth := func(status string, serverFinalError error, username, password string) {
+		t.Helper()
+
+		sc := scram.NewClient(username, "")
+		clientFirst, err := sc.ClientFirst()
+		tc.check(err, "scram clientFirst")
+		tc.client.LastTag = "x001"
+		tc.writelinef("%s authenticate scram-sha-256 %s", tc.client.LastTag, base64.StdEncoding.EncodeToString([]byte(clientFirst)))
+
+		xreadContinuation := func() []byte {
+			line, _, result, rerr := tc.client.ReadContinuation()
+			tc.check(rerr, "read continuation")
+			if result.Status != "" {
+				tc.t.Fatalf("expected continuation")
+			}
+			buf, err := base64.StdEncoding.DecodeString(line)
+			tc.check(err, "parsing base64 from remote")
+			return buf
+		}
+
+		serverFirst := xreadContinuation()
+		clientFinal, err := sc.ServerFirst(serverFirst, password)
+		tc.check(err, "scram clientFinal")
+		tc.writelinef("%s", base64.StdEncoding.EncodeToString([]byte(clientFinal)))
+
+		serverFinal := xreadContinuation()
+		err = sc.ServerFinal(serverFinal)
+		if serverFinalError == nil {
+			tc.check(err, "scram serverFinal")
+		} else if err == nil || !errors.Is(err, serverFinalError) {
+			t.Fatalf("server final, got err %#v, expected %#v", err, serverFinalError)
+		}
+		_, result, err := tc.client.Response()
+		tc.check(err, "read response")
+		if string(result.Status) != strings.ToUpper(status) {
+			tc.t.Fatalf("got status %q, expected %q", result.Status, strings.ToUpper(status))
+		}
+	}
+
+	tc = start(t)
+	auth("no", scram.ErrInvalidProof, "mjl@mox.example", "badpass")
+	auth("no", scram.ErrInvalidProof, "mjl@mox.example", "")
+	// todo: server aborts due to invalid username. we should probably make client continue with fake determinisitically generated salt and result in error in the end.
+	// auth("no", nil, "other@mox.example", "testtest")
+
+	tc.transactf("no", "authenticate bogus ")
+	tc.transactf("bad", "authenticate scram-sha-256 not base64...")
+	tc.transactf("bad", "authenticate scram-sha-256 %s", base64.StdEncoding.EncodeToString([]byte("bad data")))
+	tc.close()
+}

imapserver/copy_test.go

@@ -0,0 +1,53 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestCopy(t *testing.T) {
+	defer mockUIDValidity()()
+	tc := start(t)
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("Trash")
+
+	tc.transactf("bad", "copy")          // Missing params.
+	tc.transactf("bad", "copy 1")        // Missing params.
+	tc.transactf("bad", "copy 1 inbox ") // Leftover.
+
+	// Seqs 1,2 and UIDs 3,4.
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.StoreFlagsSet("1:2", true, `\Deleted`)
+	tc.client.Expunge()
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+
+	tc.transactf("no", "copy 1 nonexistent")
+	tc.xcode("TRYCREATE")
+
+	tc.transactf("no", "copy 1 inbox") // Cannot copy to same mailbox.
+
+	tc2.transactf("ok", "noop") // Drain.
+
+	tc.transactf("ok", "copy 1:* Trash")
+	ptr := func(v uint32) *uint32 { return &v }
+	tc.xcodeArg(imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: ptr(4)}}, To: []imapclient.NumRange{{First: 1, Last: ptr(2)}}})
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedExists(2), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchFlags(nil)}})
+
+	tc.transactf("no", "uid copy 1,2 Trash") // No match.
+	tc.transactf("ok", "uid copy 4,3 Trash")
+	tc.xcodeArg(imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: ptr(4)}}, To: []imapclient.NumRange{{First: 3, Last: ptr(4)}}})
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedExists(4), imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(3), imapclient.FetchFlags(nil)}}, imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), imapclient.FetchFlags(nil)}})
+}

imapserver/create_test.go

@@ -0,0 +1,69 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestCreate(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Login("mjl@mox.example", "testtest")
+
+	tc.transactf("no", "create inbox") // Already exists and not allowed. ../rfc/9051:1913
+	tc.transactf("no", "create Inbox") // Idem.
+
+	// ../rfc/9051:1937
+	tc.transactf("ok", "create inbox/a/c")
+	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/c"})
+
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/c"})
+
+	tc.transactf("no", "create inbox/a/c") // Exists.
+
+	tc.transactf("ok", "create inbox/a/x")
+	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/x"})
+
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "Inbox/a/x"})
+
+	// ../rfc/9051:1934
+	tc.transactf("ok", "create mailbox/")
+	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "mailbox", OldName: "mailbox/"})
+
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "mailbox"})
+
+	// If we are already subscribed, create should still work, and we still want to see the subscribed flag.
+	tc.transactf("ok", "subscribe newbox")
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`, `\NonExistent`}, Separator: '/', Mailbox: "newbox"})
+
+	tc.transactf("ok", "create newbox")
+	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "newbox"})
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "newbox"})
+
+	// todo: test create+delete+create of a name results in a higher uidvalidity.
+
+	tc.transactf("no", "create /bad/root")
+	tc.transactf("no", "create bad//root") // Cannot have internal duplicate slashes.
+	tc.transactf("no", `create ""`)        // Refuse empty mailbox name.
+	// We are not allowing special characters.
+	tc.transactf("bad", `create "\n"`)
+	tc.transactf("bad", `create "\x7f"`)
+	tc.transactf("bad", `create "\x9f"`)
+	tc.transactf("bad", `create "\u2028"`)
+	tc.transactf("bad", `create "\u2029"`)
+	tc.transactf("no", `create "%%"`)
+	tc.transactf("no", `create "*"`)
+	tc.transactf("no", `create "#"`)
+	tc.transactf("no", `create "&"`)
+}

imapserver/delete_test.go

@@ -0,0 +1,56 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestDelete(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Login("mjl@mox.example", "testtest")
+
+	tc.transactf("bad", "delete")              // Missing mailbox.
+	tc.transactf("no", "delete inbox")         // Cannot delete inbox.
+	tc.transactf("no", "delete nonexistent")   // Cannot delete mailbox that does not exist.
+	tc.transactf("no", `delete "nonexistent"`) // Again, with quoted string syntax.
+
+	tc.client.Subscribe("x")
+	tc.transactf("no", "delete x") // Subscription does not mean there is a mailbox that can be deleted.
+
+	tc.client.Create("a/b")
+	tc2.transactf("ok", "noop") // Drain changes.
+
+	// ../rfc/9051:2000
+	tc.transactf("no", "delete a") // Still has child.
+	tc.xcode("HASCHILDREN")
+
+	tc.transactf("ok", "delete a/b")
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\NonExistent`}, Separator: '/', Mailbox: "a/b"})
+
+	tc.transactf("no", "delete a/b") // Already removed.
+	tc.transactf("ok", "delete a")   // Parent can now be removed.
+	tc.transactf("ok", `list (subscribed) "" (a/b a) return (subscribed)`)
+	// Subscriptions still exist.
+	tc.xuntagged(
+		imapclient.UntaggedList{Flags: []string{`\Subscribed`, `\NonExistent`}, Separator: '/', Mailbox: "a"},
+		imapclient.UntaggedList{Flags: []string{`\Subscribed`, `\NonExistent`}, Separator: '/', Mailbox: "a/b"},
+	)
+
+	// Let's try again with a message present.
+	tc.client.Create("msgs")
+	tc.client.Append("msgs", nil, nil, []byte(exampleMsg))
+	tc.transactf("ok", "delete msgs")
+
+	// Delete for inbox/* is allowed.
+	tc.client.Create("inbox/a")
+	tc.transactf("ok", "delete inbox/a")
+
+}

imapserver/error.go

@@ -0,0 +1,55 @@
+package imapserver
+
+import (
+	"fmt"
+)
+
+func xcheckf(err error, format string, args ...any) {
+	if err != nil {
+		xserverErrorf("%s: %w", fmt.Sprintf(format, args...), err)
+	}
+}
+
+type userError struct {
+	code string // Optional response code in brackets.
+	err  error
+}
+
+func (e userError) Error() string { return e.err.Error() }
+func (e userError) Unwrap() error { return e.err }
+
+func xuserErrorf(format string, args ...any) {
+	panic(userError{err: fmt.Errorf(format, args...)})
+}
+
+func xusercodeErrorf(code, format string, args ...any) {
+	panic(userError{code: code, err: fmt.Errorf(format, args...)})
+}
+
+type serverError struct{ err error }
+
+func (e serverError) Error() string { return e.err.Error() }
+func (e serverError) Unwrap() error { return e.err }
+
+func xserverErrorf(format string, args ...any) {
+	panic(serverError{fmt.Errorf(format, args...)})
+}
+
+type syntaxError struct {
+	line string // Optional line to write before BAD result. For untagged response. CRLF will be added.
+	code string // Optional result code (between []) to write in BAD result.
+	err  error  // BAD response message.
+}
+
+func (e syntaxError) Error() string {
+	s := "bad syntax: " + e.err.Error()
+	if e.code != "" {
+		s += " [" + e.code + "]"
+	}
+	return s
+}
+func (e syntaxError) Unwrap() error { return e.err }
+
+func xsyntaxErrorf(format string, args ...any) {
+	panic(syntaxError{"", "", fmt.Errorf(format, args...)})
+}

imapserver/expunge_test.go

@@ -0,0 +1,74 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestExpunge(t *testing.T) {
+	defer mockUIDValidity()()
+	tc := start(t)
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("inbox")
+
+	tc.transactf("bad", "expunge leftover") // Leftover data.
+	tc.transactf("ok", "expunge")           // Nothing to remove though.
+	tc.xuntagged()
+
+	tc.client.Unselect()
+	tc.client.Examine("inbox")
+	tc.transactf("no", "expunge")       // Read-only.
+	tc.transactf("no", "uid expunge 1") // Read-only.
+
+	tc.client.Unselect()
+	tc.client.Select("inbox")
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.transactf("ok", "expunge") // Still nothing to remove.
+	tc.xuntagged()
+
+	tc.client.StoreFlagsAdd("1,3", true, `\Deleted`)
+
+	tc2.transactf("ok", "noop") // Drain.
+
+	tc.transactf("ok", "expunge")
+	tc.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(2))
+
+	tc2.transactf("ok", "noop") // Drain.
+	tc2.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(2))
+
+	tc.transactf("ok", "expunge") // Nothing to remove anymore.
+	tc.xuntagged()
+
+	// Only UID 2 is still left. We'll add 3 more. Getting us to UIDs 2,4,5,6.
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+
+	tc.transactf("bad", "uid expunge")            // Missing uid set.
+	tc.transactf("bad", "uid expunge 1 leftover") // Leftover data.
+	tc.transactf("bad", "uid expunge 1 leftover") // Leftover data.
+
+	tc.client.StoreFlagsAdd("1,2,4", true, `\Deleted`) // Marks UID 2,4,6 as deleted.
+
+	tc.transactf("ok", "uid expunge 1")
+	tc.xuntagged() // No match.
+
+	tc2.transactf("ok", "noop") // Drain.
+
+	tc.transactf("ok", "uid expunge 4:6") // Removes UID 4,6 at seqs 2,4.
+	tc.xuntagged(imapclient.UntaggedExpunge(2), imapclient.UntaggedExpunge(3))
+
+	tc2.transactf("ok", "noop")
+	tc.xuntagged(imapclient.UntaggedExpunge(2), imapclient.UntaggedExpunge(3))
+}

imapserver/fetch.go

@@ -0,0 +1,738 @@
+package imapserver
+
+// todo: if fetch fails part-way through the command, we wouldn't be storing the messages that were parsed. should we try harder to get parsed form of messages stored in db?
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/textproto"
+	"sort"
+	"strings"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxio"
+	"github.com/mjl-/mox/store"
+)
+
+// functions to handle fetch attribute requests are defined on fetchCmd.
+type fetchCmd struct {
+	conn          *conn
+	mailboxID     int64
+	uid           store.UID
+	tx            *bstore.Tx     // Writable tx, for storing message when first parsed as mime parts.
+	changes       []store.Change // For updated Seen flag.
+	markSeen      bool
+	needFlags     bool
+	expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
+
+	// Loaded when first needed, closed when message was processed.
+	m    *store.Message // Message currently being processed.
+	msgr *store.MsgReader
+	part *message.Part
+}
+
+// error when processing an attribute. we typically just don't respond with requested attributes that encounter a failure.
+type attrError struct{ err error }
+
+func (e attrError) Error() string {
+	return e.err.Error()
+}
+
+// raise error processing an attribute.
+func (cmd *fetchCmd) xerrorf(format string, args ...any) {
+	panic(attrError{fmt.Errorf(format, args...)})
+}
+
+func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
+	if err != nil {
+		msg := fmt.Sprintf(format, args...)
+		cmd.xerrorf("%s: %w", msg, err)
+	}
+}
+
+// Fetch returns information about messages, be it email envelopes, headers,
+// bodies, full messages, flags.
+//
+// State: Selected
+func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
+	// Command: ../rfc/9051:4330 ../rfc/3501:2992
+	// Examples: ../rfc/9051:4463 ../rfc/9051:4520
+	// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864
+
+	// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748
+	p.xspace()
+	nums := p.xnumSet()
+	p.xspace()
+	atts := p.xfetchAtts()
+	p.xempty()
+
+	// We don't use c.account.WithRLock because we write to the client while reading messages.
+	// We get the rlock, then we check the mailbox, release the lock and read the messages.
+	// The db transaction still locks out any changes to the database...
+	c.account.RLock()
+	runlock := c.account.RUnlock
+	// Note: we call runlock in a closure because we replace it below.
+	defer func() {
+		runlock()
+	}()
+
+	cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID}
+	c.xdbwrite(func(tx *bstore.Tx) {
+		cmd.tx = tx
+
+		// Ensure the mailbox still exists.
+		c.xmailboxID(tx, c.mailboxID)
+
+		uids := c.xnumSetUIDs(isUID, nums)
+
+		// Release the account lock.
+		runlock()
+		runlock = func() {} // Prevent defer from unlocking again.
+
+		for _, uid := range uids {
+			cmd.uid = uid
+			cmd.process(atts)
+		}
+	})
+
+	if len(cmd.changes) > 0 {
+		// Broadcast seen updates to other connections.
+		c.broadcast(cmd.changes)
+	}
+
+	if cmd.expungeIssued {
+		// ../rfc/2180:343
+		c.writeresultf("%s NO [EXPUNGEISSUED] at least one message was expunged", tag)
+	} else {
+		c.ok(tag, cmdstr)
+	}
+}
+
+func (cmd *fetchCmd) xensureMessage() *store.Message {
+	if cmd.m != nil {
+		return cmd.m
+	}
+
+	q := bstore.QueryTx[store.Message](cmd.tx)
+	q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
+	m, err := q.Get()
+	cmd.xcheckf(err, "get message for uid %d", cmd.uid)
+	cmd.m = &m
+	return cmd.m
+}
+
+func (cmd *fetchCmd) xensureParsed() (*store.MsgReader, *message.Part) {
+	if cmd.msgr != nil {
+		return cmd.msgr, cmd.part
+	}
+
+	m := cmd.xensureMessage()
+
+	cmd.msgr = cmd.conn.account.MessageReader(*m)
+	defer func() {
+		if cmd.part == nil {
+			err := cmd.msgr.Close()
+			cmd.conn.xsanity(err, "closing messagereader")
+			cmd.msgr = nil
+		}
+	}()
+
+	p, err := m.LoadPart(cmd.msgr)
+	xcheckf(err, "load parsed message")
+	cmd.part = &p
+	return cmd.msgr, cmd.part
+}
+
+func (cmd *fetchCmd) process(atts []fetchAtt) {
+	defer func() {
+		cmd.m = nil
+		cmd.part = nil
+		if cmd.msgr != nil {
+			err := cmd.msgr.Close()
+			cmd.conn.xsanity(err, "closing messagereader")
+			cmd.msgr = nil
+		}
+
+		x := recover()
+		if x == nil {
+			return
+		}
+		err, ok := x.(attrError)
+		if !ok {
+			panic(x)
+		}
+		if errors.Is(err, bstore.ErrAbsent) {
+			cmd.expungeIssued = true
+			return
+		}
+		cmd.conn.log.Infox("processing fetch attribute", err, mlog.Field("uid", cmd.uid))
+		xuserErrorf("processing fetch attribute: %v", err)
+	}()
+
+	data := listspace{bare("UID"), number(cmd.uid)}
+
+	cmd.markSeen = false
+	cmd.needFlags = false
+
+	for _, a := range atts {
+		data = append(data, cmd.xprocessAtt(a)...)
+	}
+
+	if cmd.markSeen {
+		m := cmd.xensureMessage()
+		m.Seen = true
+		err := cmd.tx.Update(m)
+		xcheckf(err, "marking message as seen")
+
+		cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags})
+	}
+
+	if cmd.needFlags {
+		m := cmd.xensureMessage()
+		data = append(data, bare("FLAGS"), flaglist(m.Flags))
+	}
+
+	// Write errors are turned into panics because we write through c.
+	fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
+	data.writeTo(cmd.conn, cmd.conn.bw)
+	cmd.conn.bw.Write([]byte("\r\n"))
+}
+
+// result for one attribute. if processing fails, e.g. because data was requested
+// that doesn't exist and cannot be represented in imap, the attribute is simply
+// not returned to the user. in this case, the returned value is a nil list.
+func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
+	switch a.field {
+	case "UID":
+		// Always present.
+		return nil
+	case "ENVELOPE":
+		_, part := cmd.xensureParsed()
+		envelope := xenvelope(part)
+		return []token{bare("ENVELOPE"), envelope}
+
+	case "INTERNALDATE":
+		// ../rfc/9051:6753 ../rfc/9051:6502
+		m := cmd.xensureMessage()
+		return []token{bare("INTERNALDATE"), dquote(m.Received.Format("_2-Jan-2006 15:04:05 -0700"))}
+
+	case "BODYSTRUCTURE":
+		_, part := cmd.xensureParsed()
+		bs := xbodystructure(part)
+		return []token{bare("BODYSTRUCTURE"), bs}
+
+	case "BODY":
+		respField, t := cmd.xbody(a)
+		if respField == "" {
+			return nil
+		}
+		return []token{bare(respField), t}
+
+	case "BINARY.SIZE":
+		_, p := cmd.xensureParsed()
+		if len(a.sectionBinary) == 0 {
+			// Must return the size of the entire message but with decoded body.
+			// todo: make this less expensive and/or cache the result?
+			n, err := io.Copy(io.Discard, cmd.xbinaryMessageReader(p))
+			cmd.xcheckf(err, "reading message as binary for its size")
+			return []token{bare(cmd.sectionRespField(a)), number(uint32(n))}
+		}
+		p = cmd.xpartnumsDeref(a.sectionBinary, p)
+		if len(p.Parts) > 0 || p.Message != nil {
+			// ../rfc/9051:4385
+			cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")
+		}
+		return []token{bare(cmd.sectionRespField(a)), number(p.DecodedSize)}
+
+	case "BINARY":
+		respField, t := cmd.xbinary(a)
+		if respField == "" {
+			return nil
+		}
+		return []token{bare(respField), t}
+
+	case "RFC822.SIZE":
+		m := cmd.xensureMessage()
+		return []token{bare("RFC822.SIZE"), number(m.Size)}
+
+	case "RFC822.HEADER":
+		ba := fetchAtt{
+			field: "BODY",
+			peek:  true,
+			section: &sectionSpec{
+				msgtext: &sectionMsgtext{s: "HEADER"},
+			},
+		}
+		respField, t := cmd.xbody(ba)
+		if respField == "" {
+			return nil
+		}
+		return []token{bare(a.field), t}
+
+	case "RFC822":
+		ba := fetchAtt{
+			field:   "BODY",
+			section: &sectionSpec{},
+		}
+		respField, t := cmd.xbody(ba)
+		if respField == "" {
+			return nil
+		}
+		return []token{bare(a.field), t}
+
+	case "RFC822.TEXT":
+		ba := fetchAtt{
+			field: "BODY",
+			section: &sectionSpec{
+				msgtext: &sectionMsgtext{s: "TEXT"},
+			},
+		}
+		respField, t := cmd.xbody(ba)
+		if respField == "" {
+			return nil
+		}
+		return []token{bare(a.field), t}
+
+	case "FLAGS":
+		cmd.needFlags = true
+
+	default:
+		xserverErrorf("field %q not yet implemented", a.field)
+	}
+	return nil
+}
+
+// ../rfc/9051:6522
+func xenvelope(p *message.Part) token {
+	var env message.Envelope
+	if p.Envelope != nil {
+		env = *p.Envelope
+	}
+	var date token = nilt
+	if !env.Date.IsZero() {
+		// ../rfc/5322:791
+		date = string0(env.Date.Format("Mon, 2 Jan 2006 15:04:05 -0700"))
+	}
+	var subject token = nilt
+	if env.Subject != "" {
+		subject = string0(env.Subject)
+	}
+	var inReplyTo token = nilt
+	if env.InReplyTo != "" {
+		inReplyTo = string0(env.InReplyTo)
+	}
+	var messageID token = nilt
+	if env.MessageID != "" {
+		messageID = string0(env.MessageID)
+	}
+
+	addresses := func(l []message.Address) token {
+		if len(l) == 0 {
+			return nilt
+		}
+		r := listspace{}
+		for _, a := range l {
+			var name token = nilt
+			if a.Name != "" {
+				name = string0(a.Name)
+			}
+			user := string0(a.User)
+			var host token = nilt
+			if a.Host != "" {
+				host = string0(a.Host)
+			}
+			r = append(r, listspace{name, nilt, user, host})
+		}
+		return r
+	}
+
+	// Empty sender or reply-to result in fall-back to from. ../rfc/9051:6140
+	sender := env.Sender
+	if len(sender) == 0 {
+		sender = env.From
+	}
+	replyTo := env.ReplyTo
+	if len(replyTo) == 0 {
+		replyTo = env.From
+	}
+
+	return listspace{
+		date,
+		subject,
+		addresses(env.From),
+		addresses(sender),
+		addresses(replyTo),
+		addresses(env.To),
+		addresses(env.CC),
+		addresses(env.BCC),
+		inReplyTo,
+		messageID,
+	}
+}
+
+func (cmd *fetchCmd) peekOrSeen(peek bool) {
+	if cmd.conn.readonly || peek {
+		return
+	}
+	m := cmd.xensureMessage()
+	if !m.Seen {
+		cmd.markSeen = true
+		cmd.needFlags = true
+	}
+}
+
+// reader that returns the message, but with header Content-Transfer-Encoding left out.
+func (cmd *fetchCmd) xbinaryMessageReader(p *message.Part) io.Reader {
+	hr := cmd.xmodifiedHeader(p, []string{"Content-Transfer-Encoding"}, true)
+	return io.MultiReader(hr, p.Reader())
+}
+
+// return header with only fields, or with everything except fields if "not" is set.
+func (cmd *fetchCmd) xmodifiedHeader(p *message.Part, fields []string, not bool) io.Reader {
+	h, err := io.ReadAll(p.HeaderReader())
+	cmd.xcheckf(err, "reading header")
+
+	matchesFields := func(line []byte) bool {
+		k := bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")
+		for _, f := range fields {
+			if bytes.EqualFold(k, []byte(f)) {
+				return true
+			}
+		}
+		return false
+	}
+
+	var match bool
+	hb := &bytes.Buffer{}
+	for len(h) > 0 {
+		line := h
+		i := bytes.Index(line, []byte("\r\n"))
+		if i >= 0 {
+			line = line[:i+2]
+		}
+		h = h[len(line):]
+
+		match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))
+		if match != not || len(line) == 2 {
+			hb.Write(line)
+		}
+	}
+	return hb
+}
+
+func (cmd *fetchCmd) xbinary(a fetchAtt) (string, token) {
+	_, part := cmd.xensureParsed()
+
+	cmd.peekOrSeen(a.peek)
+	if len(a.sectionBinary) == 0 {
+		r := cmd.xbinaryMessageReader(part)
+		if a.partial != nil {
+			r = cmd.xpartialReader(a.partial, r)
+		}
+		return cmd.sectionRespField(a), readerSyncliteral{r}
+	}
+
+	p := part
+	if len(a.sectionBinary) > 0 {
+		p = cmd.xpartnumsDeref(a.sectionBinary, p)
+	}
+	if len(p.Parts) != 0 || p.Message != nil {
+		// ../rfc/9051:4385
+		cmd.xerrorf("binary only allowed on leaf parts, not multipart/* or message/rfc822 or message/global")
+	}
+
+	switch p.ContentTransferEncoding {
+	case "", "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":
+	default:
+		// ../rfc/9051:5913
+		xusercodeErrorf("UNKNOWN-CTE", "unknown Content-Transfer-Encoding %q", p.ContentTransferEncoding)
+	}
+
+	r := p.Reader()
+	if a.partial != nil {
+		r = cmd.xpartialReader(a.partial, r)
+	}
+	return cmd.sectionRespField(a), readerSyncliteral{r}
+}
+
+func (cmd *fetchCmd) xpartialReader(partial *partial, r io.Reader) io.Reader {
+	n, err := io.Copy(io.Discard, io.LimitReader(r, int64(partial.offset)))
+	cmd.xcheckf(err, "skipping to offset for partial")
+	if n != int64(partial.offset) {
+		return strings.NewReader("") // ../rfc/3501:3143 ../rfc/9051:4418
+	}
+	return io.LimitReader(r, int64(partial.count))
+}
+
+func (cmd *fetchCmd) xbody(a fetchAtt) (string, token) {
+	msgr, part := cmd.xensureParsed()
+
+	if a.section == nil {
+		// Non-extensible form of BODYSTRUCTURE.
+		return a.field, xbodystructure(part)
+	}
+
+	cmd.peekOrSeen(a.peek)
+
+	respField := cmd.sectionRespField(a)
+
+	if a.section.msgtext == nil && a.section.part == nil {
+		m := cmd.xensureMessage()
+		var offset int64
+		count := m.Size
+		if a.partial != nil {
+			offset = int64(a.partial.offset)
+			if offset > m.Size {
+				offset = m.Size
+			}
+			count = int64(a.partial.count)
+			if offset+count > m.Size {
+				count = m.Size - offset
+			}
+		}
+		return respField, readerSizeSyncliteral{&moxio.AtReader{R: msgr, Offset: offset}, count}
+	}
+
+	sr := cmd.xsection(a.section, part)
+
+	if a.partial != nil {
+		n, err := io.Copy(io.Discard, io.LimitReader(sr, int64(a.partial.offset)))
+		cmd.xcheckf(err, "skipping to offset for partial")
+		if n != int64(a.partial.offset) {
+			return respField, syncliteral("") // ../rfc/3501:3143 ../rfc/9051:4418
+		}
+		return respField, readerSyncliteral{io.LimitReader(sr, int64(a.partial.count))}
+	}
+	return respField, readerSyncliteral{sr}
+}
+
+func (cmd *fetchCmd) xpartnumsDeref(nums []uint32, p *message.Part) *message.Part {
+	// ../rfc/9051:4481
+	if (len(p.Parts) == 0 && p.Message == nil) && len(nums) == 1 && nums[0] == 1 {
+		return p
+	}
+
+	// ../rfc/9051:4485
+	for i, num := range nums {
+		index := int(num - 1)
+		if p.Message != nil {
+			err := p.SetMessageReaderAt()
+			cmd.xcheckf(err, "preparing submessage")
+			return cmd.xpartnumsDeref(nums[i:], p.Message)
+		}
+		if index < 0 || index >= len(p.Parts) {
+			cmd.xerrorf("requested part does not exist")
+		}
+		p = &p.Parts[index]
+	}
+	return p
+}
+
+func (cmd *fetchCmd) xsection(section *sectionSpec, p *message.Part) io.Reader {
+	if section.part == nil {
+		return cmd.xsectionMsgtext(section.msgtext, p)
+	}
+
+	p = cmd.xpartnumsDeref(section.part.part, p)
+
+	if section.part.text == nil {
+		return p.RawReader()
+	}
+
+	// ../rfc/9051:4535
+	if p.Message != nil {
+		err := p.SetMessageReaderAt()
+		cmd.xcheckf(err, "preparing submessage")
+		p = p.Message
+	}
+
+	if !section.part.text.mime {
+		return cmd.xsectionMsgtext(section.part.text.msgtext, p)
+	}
+
+	// MIME header, see ../rfc/9051:4534 ../rfc/2045:1645
+	h, err := io.ReadAll(p.HeaderReader())
+	cmd.xcheckf(err, "reading header")
+
+	matchesFields := func(line []byte) bool {
+		k := textproto.CanonicalMIMEHeaderKey(string(bytes.TrimRight(bytes.SplitN(line, []byte(":"), 2)[0], " \t")))
+		// Only add MIME-Version and additional CRLF for messages, not other parts. ../rfc/2045:1645 ../rfc/2045:1652
+		return (p.Envelope != nil && k == "Mime-Version") || strings.HasPrefix(k, "Content-")
+	}
+
+	var match bool
+	hb := &bytes.Buffer{}
+	for len(h) > 0 {
+		line := h
+		i := bytes.Index(line, []byte("\r\n"))
+		if i >= 0 {
+			line = line[:i+2]
+		}
+		h = h[len(line):]
+
+		match = matchesFields(line) || match && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t")))
+		if match || len(line) == 2 {
+			hb.Write(line)
+		}
+	}
+	return hb
+}
+
+func (cmd *fetchCmd) xsectionMsgtext(smt *sectionMsgtext, p *message.Part) io.Reader {
+	if smt.s == "HEADER" {
+		return p.HeaderReader()
+	}
+
+	switch smt.s {
+	case "HEADER.FIELDS":
+		return cmd.xmodifiedHeader(p, smt.headers, false)
+
+	case "HEADER.FIELDS.NOT":
+		return cmd.xmodifiedHeader(p, smt.headers, true)
+
+	case "TEXT":
+		// It appears imap clients expect to get the body of the message, not a "text body"
+		// which sounds like it means a text/* part of a message. ../rfc/9051:4517
+		return p.RawReader()
+	}
+	panic(serverError{fmt.Errorf("missing case")})
+}
+
+func (cmd *fetchCmd) sectionRespField(a fetchAtt) string {
+	s := a.field + "["
+	if len(a.sectionBinary) > 0 {
+		s += fmt.Sprintf("%d", a.sectionBinary[0])
+		for _, v := range a.sectionBinary[1:] {
+			s += "." + fmt.Sprintf("%d", v)
+		}
+	} else if a.section != nil {
+		if a.section.part != nil {
+			p := a.section.part
+			s += fmt.Sprintf("%d", p.part[0])
+			for _, v := range p.part[1:] {
+				s += "." + fmt.Sprintf("%d", v)
+			}
+			if p.text != nil {
+				if p.text.mime {
+					s += ".MIME"
+				} else {
+					s += "." + cmd.sectionMsgtextName(p.text.msgtext)
+				}
+			}
+		} else if a.section.msgtext != nil {
+			s += cmd.sectionMsgtextName(a.section.msgtext)
+		}
+	}
+	s += "]"
+	// binary does not have partial in field, unlike BODY ../rfc/9051:6757
+	if a.field != "BINARY" && a.partial != nil {
+		s += fmt.Sprintf("<%d>", a.partial.offset)
+	}
+	return s
+}
+
+func (cmd *fetchCmd) sectionMsgtextName(smt *sectionMsgtext) string {
+	s := smt.s
+	if strings.HasPrefix(smt.s, "HEADER.FIELDS") {
+		l := listspace{}
+		for _, h := range smt.headers {
+			l = append(l, astring(h))
+		}
+		s += " " + l.pack(cmd.conn)
+	}
+	return s
+}
+
+func bodyFldParams(params map[string]string) token {
+	if len(params) == 0 {
+		return nilt
+	}
+	// Ensure same ordering, easier for testing.
+	var keys []string
+	for k := range params {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	l := make(listspace, 2*len(keys))
+	i := 0
+	for _, k := range keys {
+		l[i] = string0(strings.ToUpper(k))
+		l[i+1] = string0(params[k])
+		i += 2
+	}
+	return l
+}
+
+func bodyFldEnc(s string) token {
+	up := strings.ToUpper(s)
+	switch up {
+	case "7BIT", "8BIT", "BINARY", "BASE64", "QUOTED-PRINTABLE":
+		return dquote(up)
+	}
+	return string0(s)
+}
+
+// xbodystructure returns a "body".
+// calls itself for multipart messages and message/{rfc822,global}.
+func xbodystructure(p *message.Part) token {
+	if p.MediaType == "MULTIPART" {
+		// Multipart, ../rfc/9051:6355 ../rfc/9051:6411
+		var bodies concat
+		for i := range p.Parts {
+			bodies = append(bodies, xbodystructure(&p.Parts[i]))
+		}
+		return listspace{bodies, string0(p.MediaSubType)}
+	}
+
+	// ../rfc/9051:6355
+	if p.MediaType == "TEXT" {
+		// ../rfc/9051:6404 ../rfc/9051:6418
+		return listspace{
+			dquote("TEXT"), string0(p.MediaSubType), // ../rfc/9051:6739
+			// ../rfc/9051:6376
+			bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
+			nilOrString(p.ContentID),
+			nilOrString(p.ContentDescription),
+			bodyFldEnc(p.ContentTransferEncoding),
+			number(p.EndOffset - p.BodyOffset),
+			number(p.RawLineCount),
+		}
+	} else if p.MediaType == "MESSAGE" && (p.MediaSubType == "RFC822" || p.MediaSubType == "GLOBAL") {
+		// ../rfc/9051:6415
+		// note: we don't have to prepare p.Message for reading, because we aren't going to read from it.
+		return listspace{
+			dquote("MESSAGE"), dquote(p.MediaSubType), // ../rfc/9051:6732
+			// ../rfc/9051:6376
+			bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
+			nilOrString(p.ContentID),
+			nilOrString(p.ContentDescription),
+			bodyFldEnc(p.ContentTransferEncoding),
+			number(p.EndOffset - p.BodyOffset),
+			xenvelope(p.Message),
+			xbodystructure(p.Message),
+			number(p.RawLineCount), // todo: or mp.RawLineCount?
+		}
+	}
+	var media token
+	switch p.MediaType {
+	case "APPLICATION", "AUDIO", "IMAGE", "FONT", "MESSAGE", "MODEL", "VIDEO":
+		media = dquote(p.MediaType)
+	default:
+		media = string0(p.MediaType)
+	}
+	// ../rfc/9051:6404 ../rfc/9051:6407
+	return listspace{
+		media, string0(p.MediaSubType), // ../rfc/9051:6723
+		// ../rfc/9051:6376
+		bodyFldParams(p.ContentTypeParams), // ../rfc/9051:6401
+		nilOrString(p.ContentID),
+		nilOrString(p.ContentDescription),
+		bodyFldEnc(p.ContentTransferEncoding),
+		number(p.EndOffset - p.BodyOffset),
+	}
+}

imapserver/fetch_test.go

@@ -0,0 +1,403 @@
+package imapserver
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestFetch(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Enable("imap4rev2")
+	received, err := time.Parse(time.RFC3339, "2022-11-16T10:01:00+01:00")
+	tc.check(err, "parse time")
+	tc.client.Append("inbox", nil, &received, []byte(exampleMsg))
+	tc.client.Select("inbox")
+
+	uid1 := imapclient.FetchUID(1)
+	date1 := imapclient.FetchInternalDate("16-Nov-2022 10:01:00 +0100")
+	rfcsize1 := imapclient.FetchRFC822Size(len(exampleMsg))
+	env1 := imapclient.FetchEnvelope{
+		Date:      "Mon, 7 Feb 1994 21:52:25 -0800",
+		Subject:   "afternoon meeting",
+		From:      []imapclient.Address{{Name: "Fred Foobar", Mailbox: "foobar", Host: "blurdybloop.example"}},
+		Sender:    []imapclient.Address{{Name: "Fred Foobar", Mailbox: "foobar", Host: "blurdybloop.example"}},
+		ReplyTo:   []imapclient.Address{{Name: "Fred Foobar", Mailbox: "foobar", Host: "blurdybloop.example"}},
+		To:        []imapclient.Address{{Mailbox: "mooch", Host: "owatagu.siam.edu.example"}},
+		MessageID: "<B27397-0100000@Blurdybloop.example>",
+	}
+	noflags := imapclient.FetchFlags(nil)
+	bodyxstructure1 := imapclient.FetchBodystructure{
+		RespAttr: "BODY",
+		Body: imapclient.BodyTypeText{
+			MediaType:    "TEXT",
+			MediaSubtype: "PLAIN",
+			BodyFields: imapclient.BodyFields{
+				Params: [][2]string{[...]string{"CHARSET", "US-ASCII"}},
+				Octets: 57,
+			},
+			Lines: 2,
+		},
+	}
+	bodystructure1 := bodyxstructure1
+	bodystructure1.RespAttr = "BODYSTRUCTURE"
+
+	split := strings.SplitN(exampleMsg, "\r\n\r\n", 2)
+	exampleMsgHeader := split[0] + "\r\n\r\n"
+	exampleMsgBody := split[1]
+
+	binary1 := imapclient.FetchBinary{RespAttr: "BINARY[]", Data: exampleMsg}
+	binarypart1 := imapclient.FetchBinary{RespAttr: "BINARY[1]", Parts: []uint32{1}, Data: exampleMsgBody}
+	binarypartial1 := imapclient.FetchBinary{RespAttr: "BINARY[]", Data: exampleMsg[1:2]}
+	binarypartpartial1 := imapclient.FetchBinary{RespAttr: "BINARY[1]", Parts: []uint32{1}, Data: exampleMsgBody[1:2]}
+	binaryend1 := imapclient.FetchBinary{RespAttr: "BINARY[]", Data: ""}
+	binarypartend1 := imapclient.FetchBinary{RespAttr: "BINARY[1]", Parts: []uint32{1}, Data: ""}
+	binarysize1 := imapclient.FetchBinarySize{RespAttr: "BINARY.SIZE[]", Size: int64(len(exampleMsg))}
+	binarysizepart1 := imapclient.FetchBinarySize{RespAttr: "BINARY.SIZE[1]", Parts: []uint32{1}, Size: int64(len(exampleMsgBody))}
+	bodyheader1 := imapclient.FetchBody{RespAttr: "BODY[HEADER]", Section: "HEADER", Body: exampleMsgHeader}
+	bodytext1 := imapclient.FetchBody{RespAttr: "BODY[TEXT]", Section: "TEXT", Body: exampleMsgBody}
+	body1 := imapclient.FetchBody{RespAttr: "BODY[]", Body: exampleMsg}
+	bodypart1 := imapclient.FetchBody{RespAttr: "BODY[1]", Section: "1", Body: exampleMsgBody}
+	bodyoff1 := imapclient.FetchBody{RespAttr: "BODY[]<1>", Section: "", Offset: 1, Body: exampleMsg[1:3]}
+	body1off1 := imapclient.FetchBody{RespAttr: "BODY[1]<1>", Section: "1", Offset: 1, Body: exampleMsgBody[1:3]}
+	bodyend1 := imapclient.FetchBody{RespAttr: "BODY[1]<100000>", Section: "1", Offset: 100000, Body: ""} // todo: should offset be what was requested, or the size of the message?
+	rfcheader1 := imapclient.FetchRFC822Header(exampleMsgHeader)
+	rfctext1 := imapclient.FetchRFC822Text(exampleMsgBody)
+	rfc1 := imapclient.FetchRFC822(exampleMsg)
+	headerSplit := strings.SplitN(exampleMsgHeader, "\r\n", 2)
+	dateheader1 := imapclient.FetchBody{RespAttr: "BODY[HEADER.FIELDS (Date)]", Section: "HEADER.FIELDS (Date)", Body: headerSplit[0] + "\r\n\r\n"}
+	nodateheader1 := imapclient.FetchBody{RespAttr: "BODY[HEADER.FIELDS.NOT (Date)]", Section: "HEADER.FIELDS.NOT (Date)", Body: headerSplit[1]}
+	date1header1 := imapclient.FetchBody{RespAttr: "BODY[1.HEADER.FIELDS (Date)]", Section: "1.HEADER.FIELDS (Date)", Body: headerSplit[0] + "\r\n\r\n"}
+	nodate1header1 := imapclient.FetchBody{RespAttr: "BODY[1.HEADER.FIELDS.NOT (Date)]", Section: "1.HEADER.FIELDS.NOT (Date)", Body: headerSplit[1]}
+	mime1 := imapclient.FetchBody{RespAttr: "BODY[1.MIME]", Section: "1.MIME", Body: "MIME-Version: 1.0\r\nContent-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n\r\n"}
+
+	flagsSeen := imapclient.FetchFlags{`\Seen`}
+
+	tc.transactf("ok", "fetch 1 all")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1, rfcsize1, env1, noflags}})
+
+	tc.transactf("ok", "fetch 1 fast")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1, rfcsize1, noflags}})
+
+	tc.transactf("ok", "fetch 1 full")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1, rfcsize1, env1, bodyxstructure1, noflags}})
+
+	tc.transactf("ok", "fetch 1 flags")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}})
+
+	tc.transactf("ok", "fetch 1 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}})
+
+	// Should be returned unmodified, because there is no content-transfer-encoding.
+	tc.transactf("ok", "fetch 1 binary[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1, flagsSeen}})
+
+	tc.transactf("ok", "fetch 1 binary[1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypart1}}) // Seen flag not changed.
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 binary[]<1.1>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypartial1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 binary[1]<1.1>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypartpartial1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 binary[]<10000.10001>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binaryend1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 binary[1]<10000.10001>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarypartend1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 binary.size[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarysize1}})
+
+	tc.transactf("ok", "fetch 1 binary.size[1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binarysizepart1}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1, flagsSeen}})
+	tc.transactf("ok", "fetch 1 body[]<1.2>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodyoff1}}) // Already seen.
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body[1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodypart1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body[1]<1.2>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1off1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body[1]<100000.100000>")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodyend1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body[header]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodyheader1, flagsSeen}})
+
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body[text]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodytext1, flagsSeen}})
+
+	// equivalent to body.peek[header], ../rfc/3501:3183
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 rfc822.header")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfcheader1}})
+
+	// equivalent to body[text], ../rfc/3501:3199
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 rfc822.text")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfctext1, flagsSeen}})
+
+	// equivalent to body[], ../rfc/3501:3179
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 rfc822")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfc1, flagsSeen}})
+
+	// With PEEK, we should not get the \Seen flag.
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.transactf("ok", "fetch 1 body.peek[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1}})
+
+	tc.transactf("ok", "fetch 1 binary.peek[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}})
+
+	// HEADER.FIELDS and .NOT
+	tc.transactf("ok", "fetch 1 body.peek[header.fields (date)]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, dateheader1}})
+	tc.transactf("ok", "fetch 1 body.peek[header.fields.not (date)]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, nodateheader1}})
+	// For non-multipart messages, 1 means the whole message. ../rfc/9051:4481
+	tc.transactf("ok", "fetch 1 body.peek[1.header.fields (date)]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, date1header1}})
+	tc.transactf("ok", "fetch 1 body.peek[1.header.fields.not (date)]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, nodate1header1}})
+
+	// MIME, part 1 for non-multipart messages is the message itself. ../rfc/9051:4481
+	tc.transactf("ok", "fetch 1 body.peek[1.mime]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, mime1}})
+
+	// Missing sequence number. ../rfc/9051:7018
+	tc.transactf("bad", "fetch 2 body[]")
+
+	tc.transactf("ok", "fetch 1:1 body[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1, flagsSeen}})
+
+	// UID fetch
+	tc.transactf("ok", "uid fetch 1 body[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1}})
+
+	// UID fetch
+	tc.transactf("ok", "uid fetch 2 body[]")
+	tc.xuntagged()
+
+	// Test some invalid syntax.
+	tc.transactf("bad", "fetch")
+	tc.transactf("bad", "fetch ")
+	tc.transactf("bad", "fetch  ")
+	tc.transactf("bad", "fetch 1")    // At least one requested item required.
+	tc.transactf("bad", "fetch 1 ()") // Empty list not allowed
+	tc.transactf("bad", "fetch 1 unknown")
+	tc.transactf("bad", "fetch 1 (unknown)")
+	tc.transactf("bad", "fetch 1 (all)")                      // Macro's not allowed in list.
+	tc.transactf("bad", "fetch 1 binary")                     // [] required
+	tc.transactf("bad", "fetch 1 binary[text]")               // Text/header etc only allowed for body[].
+	tc.transactf("bad", "fetch 1 binary[]<1>")                // Count required.
+	tc.transactf("bad", "fetch 1 binary[]<1.0>")              // Count must be > 0.
+	tc.transactf("bad", "fetch 1 binary[]<1..1>")             // Single dot.
+	tc.transactf("bad", "fetch 1 body[]<1>")                  // Count required.
+	tc.transactf("bad", "fetch 1 body[]<1.0>")                // Count must be > 0.
+	tc.transactf("bad", "fetch 1 body[]<1..1>")               // Single dot.
+	tc.transactf("bad", "fetch 1 body[header.fields]")        // List of headers required.
+	tc.transactf("bad", "fetch 1 body[header.fields ()]")     // List must be non-empty.
+	tc.transactf("bad", "fetch 1 body[header.fields.not]")    // List of headers required.
+	tc.transactf("bad", "fetch 1 body[header.fields.not ()]") // List must be non-empty.
+	tc.transactf("bad", "fetch 1 body[mime]")                 // MIME must be prefixed with a number. ../rfc/9051:4497
+
+	tc.transactf("no", "fetch 1 body[2]") // No such part.
+
+	// Add more complex message.
+
+	uid2 := imapclient.FetchUID(2)
+	bodystructure2 := imapclient.FetchBodystructure{
+		RespAttr: "BODYSTRUCTURE",
+		Body: imapclient.BodyTypeMpart{
+			Bodies: []any{
+				imapclient.BodyTypeBasic{BodyFields: imapclient.BodyFields{Octets: 275}},
+				imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "US-ASCII"}}, Octets: 114}, Lines: 3},
+				imapclient.BodyTypeMpart{
+					Bodies: []any{
+						imapclient.BodyTypeBasic{MediaType: "AUDIO", MediaSubtype: "BASIC", BodyFields: imapclient.BodyFields{CTE: "BASE64", Octets: 22}},
+						imapclient.BodyTypeBasic{MediaType: "IMAGE", MediaSubtype: "JPEG", BodyFields: imapclient.BodyFields{CTE: "BASE64"}},
+					},
+					MediaSubtype: "PARALLEL",
+				},
+				imapclient.BodyTypeText{MediaType: "TEXT", MediaSubtype: "ENRICHED", BodyFields: imapclient.BodyFields{Octets: 145}, Lines: 5},
+				imapclient.BodyTypeMsg{
+					MediaType:    "MESSAGE",
+					MediaSubtype: "RFC822",
+					BodyFields:   imapclient.BodyFields{Octets: 228},
+					Envelope: imapclient.Envelope{
+						Subject: "(subject in US-ASCII)",
+						From:    []imapclient.Address{{Name: "", Adl: "", Mailbox: "info", Host: "mox.example"}},
+						Sender:  []imapclient.Address{{Name: "", Adl: "", Mailbox: "info", Host: "mox.example"}},
+						ReplyTo: []imapclient.Address{{Name: "", Adl: "", Mailbox: "info", Host: "mox.example"}},
+						To:      []imapclient.Address{{Name: "mox", Adl: "", Mailbox: "info", Host: "mox.example"}},
+					},
+					Bodystructure: imapclient.BodyTypeText{
+						MediaType: "TEXT", MediaSubtype: "PLAIN", BodyFields: imapclient.BodyFields{Params: [][2]string{{"CHARSET", "ISO-8859-1"}}, CTE: "QUOTED-PRINTABLE", Octets: 51}, Lines: 1},
+					Lines: 7,
+				},
+			},
+			MediaSubtype: "MIXED",
+		},
+	}
+	tc.client.Append("inbox", nil, &received, []byte(nestedMessage))
+	tc.transactf("ok", "fetch 2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	// Multiple responses.
+	tc.transactf("ok", "fetch 1:2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+	tc.transactf("ok", "fetch 1,2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+	tc.transactf("ok", "fetch 2:1 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+	tc.transactf("ok", "fetch 1:* bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+	tc.transactf("ok", "fetch *:1 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+	tc.transactf("ok", "fetch *:2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	tc.transactf("ok", "fetch * bodystructure") // Highest msgseq.
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	tc.transactf("ok", "uid fetch 1:* bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	tc.transactf("ok", "uid fetch 1:2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	tc.transactf("ok", "uid fetch 1,2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, bodystructure1}}, imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	tc.transactf("ok", "uid fetch 2:2 bodystructure")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, bodystructure2}})
+
+	// todo: read the bodies/headers of the parts, and of the nested message.
+	tc.transactf("ok", "fetch 2 body.peek[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[]", Body: nestedMessage}}})
+
+	part1 := tocrlf(`  ... Some text appears here ...
+
+[Note that the blank between the boundary and the start
+ of the text in this part means no header fields were
+ given and this is text in the US-ASCII character set.
+ It could have been done with explicit typing as in the
+ next part.]
+`)
+	tc.transactf("ok", "fetch 2 body.peek[1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[1]", Section: "1", Body: part1}}})
+
+	tc.transactf("no", "fetch 2 binary.peek[3]") // Only allowed on leaf parts, not multiparts.
+	tc.transactf("no", "fetch 2 binary.peek[5]") // Only allowed on leaf parts, not messages.
+
+	part31 := "aGVsbG8NCndvcmxkDQo=\r\n"
+	part31dec := "hello\r\nworld\r\n"
+	tc.transactf("ok", "fetch 2 binary.size[3.1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBinarySize{RespAttr: "BINARY.SIZE[3.1]", Parts: []uint32{3, 1}, Size: int64(len(part31dec))}}})
+
+	tc.transactf("ok", "fetch 2 body.peek[3.1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[3.1]", Section: "3.1", Body: part31}}})
+
+	tc.transactf("ok", "fetch 2 binary.peek[3.1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBinary{RespAttr: "BINARY[3.1]", Parts: []uint32{3, 1}, Data: part31dec}}})
+
+	part3 := tocrlf(`--unique-boundary-2
+Content-Type: audio/basic
+Content-Transfer-Encoding: base64
+
+aGVsbG8NCndvcmxkDQo=
+
+--unique-boundary-2
+Content-Type: image/jpeg
+Content-Transfer-Encoding: base64
+
+
+--unique-boundary-2--
+
+`)
+	tc.transactf("ok", "fetch 2 body.peek[3]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[3]", Section: "3", Body: part3}}})
+
+	part2mime := tocrlf(`Content-type: text/plain; charset=US-ASCII
+
+`)
+	tc.transactf("ok", "fetch 2 body.peek[2.mime]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[2.MIME]", Section: "2.MIME", Body: part2mime}}})
+
+	part5 := tocrlf(`From: info@mox.example
+To: mox <info@mox.example>
+Subject: (subject in US-ASCII)
+Content-Type: Text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: Quoted-printable
+
+  ... Additional text in ISO-8859-1 goes here ...
+`)
+	tc.transactf("ok", "fetch 2 body.peek[5]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5]", Section: "5", Body: part5}}})
+
+	part5header := tocrlf(`From: info@mox.example
+To: mox <info@mox.example>
+Subject: (subject in US-ASCII)
+Content-Type: Text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: Quoted-printable
+
+`)
+	tc.transactf("ok", "fetch 2 body.peek[5.header]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.HEADER]", Section: "5.HEADER", Body: part5header}}})
+
+	part5mime := tocrlf(`Content-Type: Text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: Quoted-printable
+
+`)
+	tc.transactf("ok", "fetch 2 body.peek[5.mime]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.MIME]", Section: "5.MIME", Body: part5mime}}})
+
+	part5text := "  ... Additional text in ISO-8859-1 goes here ...\r\n"
+	tc.transactf("ok", "fetch 2 body.peek[5.text]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.TEXT]", Section: "5.TEXT", Body: part5text}}})
+
+	tc.transactf("ok", "fetch 2 body.peek[5.1]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{uid2, imapclient.FetchBody{RespAttr: "BODY[5.1]", Section: "5.1", Body: part5text}}})
+
+	// In case of EXAMINE instead of SELECT, we should not be seeing any changed \Seen flags for non-peek commands.
+	tc.client.StoreFlagsClear("1", true, `\Seen`)
+	tc.client.Unselect()
+	tc.client.Examine("inbox")
+
+	tc.transactf("ok", "fetch 1 binary[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, binary1}})
+
+	tc.transactf("ok", "fetch 1 body[]")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, body1}})
+
+	tc.transactf("ok", "fetch 1 rfc822.text")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfctext1}})
+
+	tc.transactf("ok", "fetch 1 rfc822")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, rfc1}})
+
+	tc.client.Logout()
+}

imapserver/fuzz_test.go

@@ -0,0 +1,140 @@
+package imapserver
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/imapclient"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/store"
+)
+
+// Fuzz the server. For each fuzz string, we set up servers in various connection states, and write the string as command.
+func FuzzServer(f *testing.F) {
+	seed := []string{
+		fmt.Sprintf("authenticate plain %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000testtest"))),
+		"*",
+		"capability",
+		"noop",
+		"logout",
+		"select inbox",
+		"examine inbox",
+		"unselect",
+		"close",
+		"expunge",
+		"subscribe inbox",
+		"unsubscribe inbox",
+		`lsub "" "*"`,
+		`list "" ""`,
+		`namespace`,
+		"enable utf8=accept",
+		"create inbox",
+		"create tmpbox",
+		"rename tmpbox ntmpbox",
+		"delete ntmpbox",
+		"status inbox (uidnext messages uidvalidity deleted size unseen recent)",
+		"append inbox (\\seen) {2+}\r\nhi",
+		"fetch 1 all",
+		"fetch 1 body",
+		"fetch 1 (bodystructure)",
+		`store 1 flags (\seen \answered)`,
+		`store 1 +flags ($junk)`,
+		`store 1 -flags ($junk)`,
+		"noop",
+		"copy 1Trash",
+		"copy 1 Trash",
+		"move 1 Trash",
+		"search 1 all",
+	}
+	for _, cmd := range seed {
+		const tag = "x "
+		f.Add(tag + cmd)
+	}
+
+	mox.Context = context.Background()
+	mox.ConfigStaticPath = "../testdata/imap/mox.conf"
+	mox.MustLoadConfig()
+	dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
+	os.RemoveAll(dataDir)
+	acc, err := store.OpenAccount("mjl")
+	if err != nil {
+		f.Fatalf("open account: %v", err)
+	}
+	defer acc.Close()
+	err = acc.SetPassword("testtest")
+	if err != nil {
+		f.Fatalf("set password: %v", err)
+	}
+	done := store.Switchboard()
+	defer close(done)
+
+	comm := store.RegisterComm(acc)
+	defer comm.Unregister()
+
+	var cid int64 = 1
+
+	var fl *os.File
+	if false {
+		fl, err = os.Create("fuzz.log")
+		if err != nil {
+			f.Fatalf("fuzz log")
+		}
+		defer fl.Close()
+	}
+	flog := func(err error, msg string) {
+		if fl != nil && err != nil {
+			fmt.Fprintf(fl, "%s: %v\n", msg, err)
+		}
+	}
+
+	f.Fuzz(func(t *testing.T, s string) {
+		run := func(cmds []string) {
+			serverConn, clientConn := net.Pipe()
+			defer serverConn.Close()
+
+			go func() {
+				defer func() {
+					x := recover()
+					// Protocol can become botched, when fuzzer sends literals.
+					if x == nil {
+						return
+					}
+					err, ok := x.(error)
+					if !ok || !errors.Is(err, os.ErrDeadlineExceeded) {
+						panic(x)
+					}
+				}()
+
+				defer clientConn.Close()
+
+				err := clientConn.SetDeadline(time.Now().Add(time.Second))
+				flog(err, "set client deadline")
+				client, _ := imapclient.New(clientConn, true)
+
+				for _, cmd := range cmds {
+					client.Commandf("", "%s", cmd)
+					client.Response()
+				}
+				client.Commandf("", "%s", s)
+				client.Response()
+			}()
+
+			err = serverConn.SetDeadline(time.Now().Add(time.Second))
+			flog(err, "set server deadline")
+			serve("test", cid, nil, serverConn, false, true)
+			cid++
+		}
+
+		run([]string{})
+		run([]string{"login mjl@mox.example testtest"})
+		run([]string{"login mjl@mox.example testtest", "select inbox"})
+		xappend := fmt.Sprintf("append inbox () {%d+}\r\n%s", len(exampleMsg), exampleMsg)
+		run([]string{"login mjl@mox.example testtest", "select inbox", xappend})
+	})
+}

imapserver/idle_test.go

@@ -0,0 +1,52 @@
+package imapserver
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestIdle(t *testing.T) {
+	tc1 := start(t)
+	defer tc1.close()
+	tc1.transactf("ok", "login mjl@mox.example testtest")
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.transactf("ok", "login mjl@mox.example testtest")
+
+	tc1.transactf("ok", "select inbox")
+	tc2.transactf("ok", "select inbox")
+
+	// todo: test with delivery through smtp
+
+	tc2.cmdf("", "idle")
+	tc2.readprefixline("+")
+	done := make(chan error)
+	go func() {
+		defer func() {
+			x := recover()
+			if x != nil {
+				done <- fmt.Errorf("%v", x)
+			}
+		}()
+		untagged, _ := tc2.client.ReadUntagged()
+		var exists imapclient.UntaggedExists
+		tuntagged(tc2.t, untagged, &exists)
+		// todo: validate the data we got back.
+		tc2.writelinef("done")
+		done <- nil
+	}()
+
+	tc1.transactf("ok", "append inbox () {%d+}\r\n%s", len(exampleMsg), exampleMsg)
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case err := <-done:
+		tc1.check(err, "idle")
+	case <-timer.C:
+		t.Fatalf("idle did not finish")
+	}
+}

imapserver/list.go

@@ -0,0 +1,228 @@
+package imapserver
+
+import (
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/mjl-/bstore"
+	"github.com/mjl-/mox/store"
+)
+
+// LIST command, for listing mailboxes with various attributes, including about subscriptions and children.
+// We don't have flags Marked, Unmarked, NoSelect and NoInferiors and we don't have REMOTE mailboxes.
+//
+// State: Authenticated and selected.
+func (c *conn) cmdList(tag, cmd string, p *parser) {
+	// Command: ../rfc/9051:2224 ../rfc/6154:144 ../rfc/5258:193 ../rfc/3501:2191
+	// Examples: ../rfc/9051:2755 ../rfc/6154:347 ../rfc/5258:679 ../rfc/3501:2359
+
+	// Request syntax: ../rfc/9051:6600 ../rfc/6154:478 ../rfc/5258:1095 ../rfc/3501:4793
+	p.xspace()
+	var isExtended bool
+	var listSubscribed bool
+	var listRecursive bool
+	if p.take("(") {
+		// ../rfc/9051:6633
+		isExtended = true
+		selectOptions := map[string]bool{}
+		var nbase int
+		for !p.take(")") {
+			if len(selectOptions) > 0 {
+				p.xspace()
+			}
+			w := p.xatom()
+			W := strings.ToUpper(w)
+			switch W {
+			case "REMOTE":
+			case "RECURSIVEMATCH":
+				listRecursive = true
+			case "SUBSCRIBED":
+				nbase++
+				listSubscribed = true
+			default:
+				// ../rfc/9051:2398
+				xsyntaxErrorf("bad list selection option %q", w)
+			}
+			// Duplicates must be accepted. ../rfc/9051:2399
+			selectOptions[W] = true
+		}
+		if listRecursive && nbase == 0 {
+			// ../rfc/9051:6640
+			xsyntaxErrorf("cannot have RECURSIVEMATCH selection option without other (base) selection option")
+		}
+		p.xspace()
+	}
+	reference := p.xmailbox()
+	p.xspace()
+	patterns, isList := p.xmboxOrPat()
+	isExtended = isExtended || isList
+	var retSubscribed, retChildren, retSpecialUse bool
+	var retStatusAttrs []string
+	if p.take(" RETURN (") {
+		isExtended = true
+		// ../rfc/9051:6613 ../rfc/9051:6915 ../rfc/9051:7072 ../rfc/9051:6821 ../rfc/5819:95
+		n := 0
+		for !p.take(")") {
+			if n > 0 {
+				p.xspace()
+			}
+			n++
+			w := p.xatom()
+			W := strings.ToUpper(w)
+			switch W {
+			case "SUBSCRIBED":
+				retSubscribed = true
+			case "CHILDREN":
+				// ../rfc/3348:44
+				retChildren = true
+			case "SPECIAL-USE":
+				// ../rfc/6154:478
+				retSpecialUse = true
+			case "STATUS":
+				// ../rfc/9051:7072 ../rfc/5819:181
+				p.xspace()
+				p.xtake("(")
+				retStatusAttrs = []string{p.xstatusAtt()}
+				for p.take(" ") {
+					retStatusAttrs = append(retStatusAttrs, p.xstatusAtt())
+				}
+				p.xtake(")")
+			default:
+				// ../rfc/9051:2398
+				xsyntaxErrorf("bad list return option %q", w)
+			}
+		}
+	}
+	p.xempty()
+
+	if !isExtended && reference == "" && patterns[0] == "" {
+		// ../rfc/9051:2277 ../rfc/3501:2221
+		c.bwritelinef(`* LIST () "/" ""`)
+		c.ok(tag, cmd)
+		return
+	}
+
+	if isExtended {
+		// ../rfc/9051:2286
+		n := make([]string, 0, len(patterns))
+		for _, p := range patterns {
+			if p != "" {
+				n = append(n, p)
+			}
+		}
+		patterns = n
+	}
+	re := xmailboxPatternMatcher(reference, patterns)
+	var responseLines []string
+
+	c.account.WithRLock(func() {
+		c.xdbread(func(tx *bstore.Tx) {
+			type info struct {
+				mailbox    *store.Mailbox
+				subscribed bool
+			}
+			names := map[string]info{}
+			hasSubscribedChild := map[string]bool{}
+			hasChild := map[string]bool{}
+			var nameList []string
+
+			q := bstore.QueryTx[store.Mailbox](tx)
+			err := q.ForEach(func(mb store.Mailbox) error {
+				names[mb.Name] = info{mailbox: &mb}
+				nameList = append(nameList, mb.Name)
+				for p := filepath.Dir(mb.Name); p != "."; p = filepath.Dir(p) {
+					hasChild[p] = true
+				}
+				return nil
+			})
+			xcheckf(err, "listing mailboxes")
+
+			qs := bstore.QueryTx[store.Subscription](tx)
+			err = qs.ForEach(func(sub store.Subscription) error {
+				info, ok := names[sub.Name]
+				info.subscribed = true
+				names[sub.Name] = info
+				if !ok {
+					nameList = append(nameList, sub.Name)
+				}
+				for p := filepath.Dir(sub.Name); p != "."; p = filepath.Dir(p) {
+					hasSubscribedChild[p] = true
+				}
+				return nil
+			})
+			xcheckf(err, "listing subscriptions")
+
+			sort.Strings(nameList) // For predictable order in tests.
+
+			for _, name := range nameList {
+				if !re.MatchString(name) {
+					continue
+				}
+				info := names[name]
+
+				var flags listspace
+				var extended listspace
+				if listRecursive && hasSubscribedChild[name] {
+					extended = listspace{bare("CHILDINFO"), listspace{dquote("SUBSCRIBED")}}
+				}
+				if listSubscribed && info.subscribed {
+					flags = append(flags, bare(`\Subscribed`))
+					if info.mailbox == nil {
+						flags = append(flags, bare(`\NonExistent`))
+					}
+				}
+				if (info.mailbox == nil || listSubscribed) && flags == nil && extended == nil {
+					continue
+				}
+
+				if retChildren {
+					var f string
+					if hasChild[name] {
+						f = `\HasChildren`
+					} else {
+						f = `\HasNoChildren`
+					}
+					flags = append(flags, bare(f))
+				}
+				if !listSubscribed && retSubscribed && info.subscribed {
+					flags = append(flags, bare(`\Subscribed`))
+				}
+				if retSpecialUse && info.mailbox != nil {
+					if info.mailbox.Archive {
+						flags = append(flags, bare(`\Archive`))
+					}
+					if info.mailbox.Draft {
+						flags = append(flags, bare(`\Draft`))
+					}
+					if info.mailbox.Junk {
+						flags = append(flags, bare(`\Junk`))
+					}
+					if info.mailbox.Sent {
+						flags = append(flags, bare(`\Sent`))
+					}
+					if info.mailbox.Trash {
+						flags = append(flags, bare(`\Trash`))
+					}
+				}
+
+				var extStr string
+				if extended != nil {
+					extStr = " " + extended.pack(c)
+				}
+				line := fmt.Sprintf(`* LIST %s "/" %s%s`, flags.pack(c), astring(name).pack(c), extStr)
+				responseLines = append(responseLines, line)
+
+				if retStatusAttrs != nil && info.mailbox != nil {
+					responseLines = append(responseLines, c.xstatusLine(tx, *info.mailbox, retStatusAttrs))
+				}
+			}
+		})
+	})
+
+	for _, line := range responseLines {
+		c.bwritelinef("%s", line)
+	}
+	c.ok(tag, cmd)
+}

imapserver/list_test.go

@@ -0,0 +1,215 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+	"github.com/mjl-/mox/store"
+)
+
+func TestListBasic(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	ulist := func(name string, flags ...string) imapclient.UntaggedList {
+		if len(flags) == 0 {
+			flags = nil
+		}
+		return imapclient.UntaggedList{Flags: flags, Separator: '/', Mailbox: name}
+	}
+
+	tc.last(tc.client.List("INBOX"))
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.last(tc.client.List("Inbox"))
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.last(tc.client.List("%"))
+	tc.xuntagged(ulist("Archive"), ulist("Drafts"), ulist("Inbox"), ulist("Junk"), ulist("Sent"), ulist("Trash"))
+
+	tc.last(tc.client.List("*"))
+	tc.xuntagged(ulist("Archive"), ulist("Drafts"), ulist("Inbox"), ulist("Junk"), ulist("Sent"), ulist("Trash"))
+
+	tc.last(tc.client.List("A*"))
+	tc.xuntagged(ulist("Archive"))
+
+	tc.client.Create("Inbox/todo")
+
+	tc.last(tc.client.List("Inbox*"))
+	tc.xuntagged(ulist("Inbox"), ulist("Inbox/todo"))
+
+	tc.last(tc.client.List("Inbox/%"))
+	tc.xuntagged(ulist("Inbox/todo"))
+
+	tc.last(tc.client.List("Inbox/*"))
+	tc.xuntagged(ulist("Inbox/todo"))
+
+	// Leading full INBOX is turned into Inbox, so mailbox matches.
+	tc.last(tc.client.List("INBOX/*"))
+	tc.xuntagged(ulist("Inbox/todo"))
+
+	// No match because we are only touching various casings of the full "INBOX".
+	tc.last(tc.client.List("INBO*"))
+	tc.xuntagged()
+}
+
+func TestListExtended(t *testing.T) {
+	defer mockUIDValidity()()
+
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	ulist := func(name string, flags ...string) imapclient.UntaggedList {
+		if len(flags) == 0 {
+			flags = nil
+		}
+		return imapclient.UntaggedList{Flags: flags, Separator: '/', Mailbox: name}
+	}
+
+	uidvals := map[string]uint32{}
+	for _, name := range store.InitialMailboxes {
+		uidvals[name] = 1
+	}
+	var uidvalnext uint32 = 2
+	uidval := func(name string) uint32 {
+		v, ok := uidvals[name]
+		if !ok {
+			v = uidvalnext
+			uidvals[name] = v
+			uidvalnext++
+		}
+		return v
+	}
+
+	ustatus := func(name string) imapclient.UntaggedStatus {
+		attrs := map[string]int64{
+			"MESSAGES":    0,
+			"UIDNEXT":     1,
+			"UIDVALIDITY": int64(uidval(name)),
+			"UNSEEN":      0,
+			"DELETED":     0,
+			"SIZE":        0,
+			"RECENT":      0,
+			"APPENDLIMIT": 0,
+		}
+		return imapclient.UntaggedStatus{Mailbox: name, Attrs: attrs}
+	}
+
+	const (
+		Fsubscribed    = `\Subscribed`
+		Fhaschildren   = `\HasChildren`
+		Fhasnochildren = `\HasNoChildren`
+		Fnonexistent   = `\NonExistent`
+		Farchive       = `\Archive`
+		Fdraft         = `\Draft`
+		Fjunk          = `\Junk`
+		Fsent          = `\Sent`
+		Ftrash         = `\Trash`
+	)
+
+	// untaggedlist with flags subscribed and hasnochildren
+	xlist := func(name string, flags ...string) imapclient.UntaggedList {
+		flags = append([]string{Fhasnochildren, Fsubscribed}, flags...)
+		return ulist(name, flags...)
+	}
+
+	xchildlist := func(name string, flags ...string) imapclient.UntaggedList {
+		u := ulist(name, flags...)
+		comp := imapclient.TaggedExtComp{String: "SUBSCRIBED"}
+		u.Extended = []imapclient.MboxListExtendedItem{{Tag: "CHILDINFO", Val: imapclient.TaggedExtVal{Comp: &comp}}}
+		return u
+	}
+
+	tc.last(tc.client.ListFull(false, "INBOX"))
+	tc.xuntagged(xlist("Inbox"), ustatus("Inbox"))
+
+	tc.last(tc.client.ListFull(false, "Inbox"))
+	tc.xuntagged(xlist("Inbox"), ustatus("Inbox"))
+
+	tc.last(tc.client.ListFull(false, "%"))
+	tc.xuntagged(xlist("Archive", Farchive), ustatus("Archive"), xlist("Drafts", Fdraft), ustatus("Drafts"), xlist("Inbox"), ustatus("Inbox"), xlist("Junk", Fjunk), ustatus("Junk"), xlist("Sent", Fsent), ustatus("Sent"), xlist("Trash", Ftrash), ustatus("Trash"))
+
+	tc.last(tc.client.ListFull(false, "*"))
+	tc.xuntagged(xlist("Archive", Farchive), ustatus("Archive"), xlist("Drafts", Fdraft), ustatus("Drafts"), xlist("Inbox"), ustatus("Inbox"), xlist("Junk", Fjunk), ustatus("Junk"), xlist("Sent", Fsent), ustatus("Sent"), xlist("Trash", Ftrash), ustatus("Trash"))
+
+	tc.last(tc.client.ListFull(false, "A*"))
+	tc.xuntagged(xlist("Archive", Farchive), ustatus("Archive"))
+
+	tc.last(tc.client.ListFull(false, "A*", "Junk"))
+	tc.xuntagged(xlist("Archive", Farchive), ustatus("Archive"), xlist("Junk", Fjunk), ustatus("Junk"))
+
+	tc.client.Create("Inbox/todo")
+
+	tc.last(tc.client.ListFull(false, "Inbox*"))
+	tc.xuntagged(ulist("Inbox", Fhaschildren, Fsubscribed), ustatus("Inbox"), xlist("Inbox/todo"), ustatus("Inbox/todo"))
+
+	tc.last(tc.client.ListFull(false, "Inbox/%"))
+	tc.xuntagged(xlist("Inbox/todo"), ustatus("Inbox/todo"))
+
+	tc.last(tc.client.ListFull(false, "Inbox/*"))
+	tc.xuntagged(xlist("Inbox/todo"), ustatus("Inbox/todo"))
+
+	// Leading full INBOX is turned into Inbox, so mailbox matches.
+	tc.last(tc.client.ListFull(false, "INBOX/*"))
+	tc.xuntagged(xlist("Inbox/todo"), ustatus("Inbox/todo"))
+
+	// No match because we are only touching various casings of the full "INBOX".
+	tc.last(tc.client.ListFull(false, "INBO*"))
+	tc.xuntagged()
+
+	tc.last(tc.client.ListFull(true, "Inbox"))
+	tc.xuntagged(xchildlist("Inbox", Fsubscribed, Fhaschildren), ustatus("Inbox"))
+
+	tc.client.Unsubscribe("Inbox")
+	tc.last(tc.client.ListFull(true, "Inbox"))
+	tc.xuntagged(xchildlist("Inbox", Fhaschildren), ustatus("Inbox"))
+
+	tc.client.Delete("Inbox/todo") // Still subscribed.
+	tc.last(tc.client.ListFull(true, "Inbox"))
+	tc.xuntagged(xchildlist("Inbox", Fhasnochildren), ustatus("Inbox"))
+
+	// Simple extended list without RETURN options.
+	tc.transactf("ok", `list "" ("inbox")`)
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.transactf("ok", `list () "" ("inbox") return ()`)
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.transactf("ok", `list "" ("inbox") return ()`)
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.transactf("ok", `list () "" ("inbox")`)
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.transactf("ok", `list (remote) "" ("inbox")`)
+	tc.xuntagged(ulist("Inbox"))
+
+	tc.transactf("ok", `list (remote) "" "/inbox"`)
+	tc.xuntagged()
+
+	tc.transactf("ok", `list (remote) "/inbox" ""`)
+	tc.xuntagged()
+
+	tc.transactf("ok", `list (remote) "inbox" ""`)
+	tc.xuntagged()
+
+	tc.transactf("ok", `list (remote) "inbox" "a"`)
+	tc.xuntagged()
+
+	tc.client.Create("inbox/a")
+	tc.transactf("ok", `list (remote) "inbox" "a"`)
+	tc.xuntagged(ulist("Inbox/a"))
+
+	tc.client.Subscribe("x")
+	tc.transactf("ok", `list (subscribed) "" x return (subscribed)`)
+	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`, `\NonExistent`}, Separator: '/', Mailbox: "x"})
+
+	tc.transactf("bad", `list (recursivematch) "" "*"`)        // Cannot have recursivematch without a base selection option like subscribed.
+	tc.transactf("bad", `list (recursivematch remote) "" "*"`) // "remote" is not a base selection option.
+	tc.transactf("bad", `list (unknown) "" "*"`)               // Unknown selection options must result in BAD.
+	tc.transactf("bad", `list () "" "*" return (unknown)`)     // Unknown return options must result in BAD.
+}

imapserver/lsub_test.go

@@ -0,0 +1,35 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestLsub(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	tc.transactf("bad", "lsub")       // Missing params.
+	tc.transactf("bad", `lsub ""`)    // Missing param.
+	tc.transactf("bad", `lsub "" x `) // Leftover data.
+
+	tc.transactf("ok", `lsub "" x*`)
+	tc.xuntagged()
+
+	tc.transactf("ok", "create a/b/c")
+	tc.transactf("ok", `lsub "" a/*`)
+	tc.xuntagged(imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b"}, imapclient.UntaggedLsub{Separator: '/', Mailbox: "a/b/c"})
+
+	// ../rfc/3501:2394
+	tc.transactf("ok", "unsubscribe a")
+	tc.transactf("ok", "unsubscribe a/b")
+	tc.transactf("ok", `lsub "" a/%%`)
+	tc.xuntagged(imapclient.UntaggedLsub{Flags: []string{`\NoSelect`}, Separator: '/', Mailbox: "a/b"})
+
+	tc.transactf("ok", "unsubscribe a/b/c")
+	tc.transactf("ok", `lsub "" a/%%`)
+	tc.xuntagged()
+}

imapserver/move_test.go

@@ -0,0 +1,92 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestMove(t *testing.T) {
+	defer mockUIDValidity()()
+	tc := start(t)
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+
+	tc3 := startNoSwitchboard(t)
+	defer tc3.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("Trash")
+
+	tc3.client.Login("mjl@mox.example", "testtest")
+	tc3.client.Select("inbox")
+
+	tc.transactf("bad", "move")          // Missing params.
+	tc.transactf("bad", "move 1")        // Missing params.
+	tc.transactf("bad", "move 1 inbox ") // Leftover.
+
+	// Seqs 1,2 and UIDs 3,4.
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.StoreFlagsSet("1:2", true, `\Deleted`)
+	tc.client.Expunge()
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+
+	tc.client.Unselect()
+	tc.client.Examine("inbox")
+	tc.transactf("no", "move 1 Trash") // Opened readonly.
+	tc.client.Unselect()
+	tc.client.Select("inbox")
+
+	tc.transactf("no", "move 1 nonexistent")
+	tc.xcode("TRYCREATE")
+
+	tc.transactf("no", "move 1 inbox") // Cannot move to same mailbox.
+
+	tc2.transactf("ok", "noop") // Drain.
+	tc3.transactf("ok", "noop") // Drain.
+
+	tc.transactf("ok", "move 1:* Trash")
+	ptr := func(v uint32) *uint32 { return &v }
+	tc.xuntagged(
+		imapclient.UntaggedResult{Status: "OK", RespText: imapclient.RespText{Code: "COPYUID", CodeArg: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 3, Last: ptr(4)}}, To: []imapclient.NumRange{{First: 1, Last: ptr(2)}}}, More: "moved"}},
+		imapclient.UntaggedExpunge(1),
+		imapclient.UntaggedExpunge(1),
+	)
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(
+		imapclient.UntaggedExists(2),
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchFlags(nil)}},
+	)
+	tc3.transactf("ok", "noop")
+	tc3.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1))
+
+	// UIDs 5,6
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc2.transactf("ok", "noop") // Drain.
+	tc3.transactf("ok", "noop") // Drain.
+
+	tc.transactf("no", "uid move 1:4 Trash") // No match.
+	tc.transactf("ok", "uid move 6:5 Trash")
+	tc.xuntagged(
+		imapclient.UntaggedResult{Status: "OK", RespText: imapclient.RespText{Code: "COPYUID", CodeArg: imapclient.CodeCopyUID{DestUIDValidity: 1, From: []imapclient.NumRange{{First: 5, Last: ptr(6)}}, To: []imapclient.NumRange{{First: 3, Last: ptr(4)}}}, More: "moved"}},
+		imapclient.UntaggedExpunge(1),
+		imapclient.UntaggedExpunge(1),
+	)
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(
+		imapclient.UntaggedExists(4),
+		imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(3), imapclient.FetchFlags(nil)}},
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), imapclient.FetchFlags(nil)}},
+	)
+	tc3.transactf("ok", "noop")
+	tc3.xuntagged(imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1))
+}

imapserver/pack.go

@@ -0,0 +1,213 @@
+package imapserver
+
+import (
+	"fmt"
+	"io"
+)
+
+type token interface {
+	pack(c *conn) string
+	writeTo(c *conn, w io.Writer)
+}
+
+type bare string
+
+func (t bare) pack(c *conn) string {
+	return string(t)
+}
+
+func (t bare) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}
+
+type niltoken struct{}
+
+var nilt niltoken
+
+func (t niltoken) pack(c *conn) string {
+	return "NIL"
+}
+
+func (t niltoken) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}
+
+func nilOrString(s string) token {
+	if s == "" {
+		return nilt
+	}
+	return string0(s)
+}
+
+type string0 string
+
+// ../rfc/9051:7081
+// ../rfc/9051:6856 ../rfc/6855:153
+func (t string0) pack(c *conn) string {
+	r := `"`
+	for _, ch := range t {
+		if ch == '\x00' || ch == '\r' || ch == '\n' || ch > 0x7f && !c.utf8strings() {
+			return syncliteral(t).pack(c)
+		}
+		if ch == '\\' || ch == '"' {
+			r += `\`
+		}
+		r += string(ch)
+	}
+	r += `"`
+	return r
+}
+
+func (t string0) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}
+
+type dquote string
+
+func (t dquote) pack(c *conn) string {
+	r := `"`
+	for _, c := range t {
+		if c == '\\' || c == '"' {
+			r += `\`
+		}
+		r += string(c)
+	}
+	r += `"`
+	return r
+}
+
+func (t dquote) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}
+
+type syncliteral string
+
+func (t syncliteral) pack(c *conn) string {
+	return fmt.Sprintf("{%d}\r\n", len(t)) + string(t)
+}
+
+func (t syncliteral) writeTo(c *conn, w io.Writer) {
+	fmt.Fprintf(w, "{%d}\r\n", len(t))
+	w.Write([]byte(t))
+}
+
+// data from reader with known size.
+type readerSizeSyncliteral struct {
+	r    io.Reader
+	size int64
+}
+
+func (t readerSizeSyncliteral) pack(c *conn) string {
+	buf, err := io.ReadAll(t.r)
+	if err != nil {
+		panic(err)
+	}
+	return fmt.Sprintf("{%d}\r\n", t.size) + string(buf)
+}
+
+func (t readerSizeSyncliteral) writeTo(c *conn, w io.Writer) {
+	fmt.Fprintf(w, "{%d}\r\n", t.size)
+	if _, err := io.Copy(w, io.LimitReader(t.r, t.size)); err != nil {
+		panic(err)
+	}
+}
+
+// data from reader without known size.
+type readerSyncliteral struct {
+	r io.Reader
+}
+
+func (t readerSyncliteral) pack(c *conn) string {
+	buf, err := io.ReadAll(t.r)
+	if err != nil {
+		panic(err)
+	}
+	return fmt.Sprintf("{%d}\r\n", len(buf)) + string(buf)
+}
+
+func (t readerSyncliteral) writeTo(c *conn, w io.Writer) {
+	buf, err := io.ReadAll(t.r)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Fprintf(w, "{%d}\r\n", len(buf))
+	_, err = w.Write(buf)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// list with tokens space-separated
+type listspace []token
+
+func (t listspace) pack(c *conn) string {
+	s := "("
+	for i, e := range t {
+		if i > 0 {
+			s += " "
+		}
+		s += e.pack(c)
+	}
+	s += ")"
+	return s
+}
+
+func (t listspace) writeTo(c *conn, w io.Writer) {
+	fmt.Fprint(w, "(")
+	for i, e := range t {
+		if i > 0 {
+			fmt.Fprint(w, " ")
+		}
+		e.writeTo(c, w)
+	}
+	fmt.Fprint(w, ")")
+}
+
+// Concatenated tokens, no spaces or list syntax.
+type concat []token
+
+func (t concat) pack(c *conn) string {
+	var s string
+	for _, e := range t {
+		s += e.pack(c)
+	}
+	return s
+}
+
+func (t concat) writeTo(c *conn, w io.Writer) {
+	for _, e := range t {
+		e.writeTo(c, w)
+	}
+}
+
+type astring string
+
+func (t astring) pack(c *conn) string {
+	if len(t) == 0 {
+		return string0(t).pack(c)
+	}
+next:
+	for _, ch := range t {
+		for _, x := range atomChar {
+			if ch == x {
+				continue next
+			}
+		}
+		return string0(t).pack(c)
+	}
+	return string(t)
+}
+
+func (t astring) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}
+
+type number uint32
+
+func (t number) pack(c *conn) string {
+	return fmt.Sprintf("%d", t)
+}
+
+func (t number) writeTo(c *conn, w io.Writer) {
+	w.Write([]byte(t.pack(c)))
+}

imapserver/parse.go

@@ -0,0 +1,942 @@
+package imapserver
+
+import (
+	"fmt"
+	"net/textproto"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+var (
+	listWildcards = "%*"
+	char          = charRange('\x01', '\x7f')
+	ctl           = charRange('\x01', '\x19')
+	atomChar      = charRemove(char, "(){ "+listWildcards+ctl)
+	respSpecials  = atomChar + "]"
+	astringChar   = atomChar + respSpecials
+)
+
+func charRange(first, last rune) string {
+	r := ""
+	c := first
+	r += string(c)
+	for c < last {
+		c++
+		r += string(c)
+	}
+	return r
+}
+
+func charRemove(s, remove string) string {
+	r := ""
+next:
+	for _, c := range s {
+		for _, x := range remove {
+			if c == x {
+				continue next
+			}
+		}
+		r += string(c)
+	}
+	return r
+}
+
+type parser struct {
+	// Orig is the line in original casing, and upper in upper casing. We often match
+	// against upper for easy case insensitive handling as IMAP requires, but sometimes
+	// return from orig to keep the original case.
+	orig     string
+	upper    string
+	o        int      // Current offset in parsing.
+	contexts []string // What we're parsing, for error messages.
+	conn     *conn
+}
+
+// toUpper upper cases bytes that are a-z. strings.ToUpper does too much. and
+// would replace invalid bytes with unicode replacement characters, which would
+// break our requirement that offsets into the original and upper case strings
+// point to the same character.
+func toUpper(s string) string {
+	r := []byte(s)
+	for i, c := range r {
+		if c >= 'a' && c <= 'z' {
+			r[i] = c - 0x20
+		}
+	}
+	return string(r)
+}
+
+func newParser(s string, conn *conn) *parser {
+	return &parser{s, toUpper(s), 0, nil, conn}
+}
+
+func (p *parser) xerrorf(format string, args ...any) {
+	var context string
+	if len(p.contexts) > 0 {
+		context = strings.Join(p.contexts, ",")
+	}
+	panic(syntaxError{"", "", fmt.Errorf("%s (%sremaining data %q)", fmt.Sprintf(format, args...), context, p.orig[p.o:])})
+}
+
+func (p *parser) context(s string) func() {
+	p.contexts = append(p.contexts, s)
+	return func() {
+		p.contexts = p.contexts[:len(p.contexts)-1]
+	}
+}
+
+func (p *parser) empty() bool {
+	return p.o == len(p.upper)
+}
+
+func (p *parser) xempty() {
+	if !p.empty() {
+		p.xerrorf("leftover data")
+	}
+}
+
+func (p *parser) hasPrefix(s string) bool {
+	return strings.HasPrefix(p.upper[p.o:], s)
+}
+
+func (p *parser) take(s string) bool {
+	if !p.hasPrefix(s) {
+		return false
+	}
+	p.o += len(s)
+	return true
+}
+
+func (p *parser) xtake(s string) {
+	if !p.take(s) {
+		p.xerrorf("expected %q", s)
+	}
+}
+
+func (p *parser) xnonempty() {
+	if p.empty() {
+		p.xerrorf("unexpected end")
+	}
+}
+
+func (p *parser) xtakeall() string {
+	r := p.orig[p.o:]
+	p.o = len(p.orig)
+	return r
+}
+
+func (p *parser) xtake1n(n int, what string) string {
+	if n == 0 {
+		p.xerrorf("expected chars from %s", what)
+	}
+	return p.xtaken(n)
+}
+
+func (p *parser) xtake1fn(fn func(i int, c rune) bool) string {
+	i := 0
+	s := ""
+	for _, c := range p.upper[p.o:] {
+		if !fn(i, c) {
+			break
+		}
+		s += string(c)
+		i++
+	}
+	if s == "" {
+		p.xerrorf("expected at least one character")
+	}
+	p.o += len(s)
+	return s
+}
+
+func (p *parser) xtakechars(s string, what string) string {
+	p.xnonempty()
+	for i, c := range p.orig[p.o:] {
+		if !contains(s, c) {
+			return p.xtake1n(i, what)
+		}
+	}
+	return p.xtakeall()
+}
+
+func (p *parser) xtaken(n int) string {
+	if p.o+n > len(p.orig) {
+		p.xerrorf("not enough data")
+	}
+	r := p.orig[p.o : p.o+n]
+	p.o += n
+	return r
+}
+
+func (p *parser) peekn(n int) (string, bool) {
+	if len(p.upper[p.o:]) < n {
+		return "", false
+	}
+	return p.upper[p.o : p.o+n], true
+}
+
+func (p *parser) space() bool {
+	return p.take(" ")
+}
+
+func (p *parser) xspace() {
+	if !p.space() {
+		p.xerrorf("expected space")
+	}
+}
+
+func (p *parser) digits() string {
+	var n int
+	for _, c := range p.upper[p.o:] {
+		if c >= '0' && c <= '9' {
+			n++
+		}
+	}
+	if n == 0 {
+		return ""
+	}
+	s := p.upper[p.o : p.o+n]
+	p.o += n
+	return s
+}
+
+func (p *parser) nznumber() (uint32, bool) {
+	o := p.o
+	for o < len(p.upper) && p.upper[o] >= '0' && p.upper[o] <= '9' {
+		o++
+	}
+	if o == p.o {
+		return 0, false
+	}
+	if n, err := strconv.ParseUint(p.upper[p.o:o], 10, 32); err != nil {
+		return 0, false
+	} else if n == 0 {
+		return 0, false
+	} else {
+		p.o = o
+		return uint32(n), true
+	}
+}
+
+func (p *parser) xnznumber() uint32 {
+	n, ok := p.nznumber()
+	if !ok {
+		p.xerrorf("expected non-zero number")
+	}
+	return n
+}
+
+func (p *parser) number() (uint32, bool) {
+	o := p.o
+	for o < len(p.upper) && p.upper[o] >= '0' && p.upper[o] <= '9' {
+		o++
+	}
+	if o == p.o {
+		return 0, false
+	}
+	n, err := strconv.ParseUint(p.upper[p.o:o], 10, 32)
+	if err != nil {
+		return 0, false
+	}
+	p.o = o
+	return uint32(n), true
+}
+
+func (p *parser) xnumber() uint32 {
+	n, ok := p.number()
+	if !ok {
+		p.xerrorf("expected number")
+	}
+	return n
+}
+
+func (p *parser) xnumber64() int64 {
+	s := p.digits()
+	if s == "" {
+		p.xerrorf("expected number64")
+	}
+	v, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		p.xerrorf("parsing number64 %q: %v", s, err)
+	}
+	return v
+}
+
+// l should be a list of uppercase words, the first match is returned
+func (p *parser) takelist(l ...string) (string, bool) {
+	for _, w := range l {
+		if p.take(w) {
+			return w, true
+		}
+	}
+	return "", false
+}
+
+func (p *parser) xtakelist(l ...string) string {
+	w, ok := p.takelist(l...)
+	if !ok {
+		p.xerrorf("expected one of %s", strings.Join(l, ","))
+	}
+	return w
+}
+
+func (p *parser) xstring() (r string) {
+	if p.take(`"`) {
+		esc := false
+		r := ""
+		for i, c := range p.orig[p.o:] {
+			if c == '\\' {
+				esc = true
+			} else if c == '\x00' || c == '\r' || c == '\n' {
+				p.xerrorf("invalid nul, cr or lf in string")
+			} else if esc {
+				if c == '\\' || c == '"' {
+					r += string(c)
+					esc = false
+				} else {
+					p.xerrorf("invalid escape char %c", c)
+				}
+			} else if c == '"' {
+				p.o += i + 1
+				return r
+			} else {
+				r += string(c)
+			}
+		}
+		p.xerrorf("missing closing dquote in string")
+	}
+	size, sync := p.xliteralSize(100*1024, false)
+	s := p.conn.xreadliteral(size, sync)
+	line := p.conn.readline(false)
+	p.orig, p.upper, p.o = line, toUpper(line), 0
+	return s
+}
+
+func (p *parser) xnil() {
+	p.xtake("NIL")
+}
+
+// Returns NIL as empty string.
+func (p *parser) xnilString() string {
+	if p.take("NIL") {
+		return ""
+	}
+	return p.xstring()
+}
+
+func (p *parser) xastring() string {
+	if p.hasPrefix(`"`) || p.hasPrefix("{") || p.hasPrefix("~{") {
+		return p.xstring()
+	}
+	return p.xtakechars(astringChar, "astring")
+}
+
+func contains(s string, c rune) bool {
+	for _, x := range s {
+		if x == c {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *parser) xtag() string {
+	p.xnonempty()
+	for i, c := range p.orig[p.o:] {
+		if c == '+' || !contains(astringChar, c) {
+			return p.xtake1n(i, "tag")
+		}
+	}
+	return p.xtakeall()
+}
+
+func (p *parser) xcommand() string {
+	for i, c := range p.upper[p.o:] {
+		if !(c >= 'A' && c <= 'Z' || c == ' ' && p.upper[p.o:p.o+i] == "UID") {
+			return p.xtake1n(i, "command")
+		}
+	}
+	return p.xtakeall()
+}
+
+func (p *parser) remainder() string {
+	return p.orig[p.o:]
+}
+
+func (p *parser) xflag() string {
+	return p.xtakelist(`\`, "$") + p.xatom()
+}
+
+func (p *parser) xflagList() (l []string) {
+	p.xtake("(")
+	if !p.hasPrefix(")") {
+		l = append(l, p.xflag())
+	}
+	for !p.take(")") {
+		p.xspace()
+		l = append(l, p.xflag())
+	}
+	return
+}
+
+func (p *parser) xatom() string {
+	return p.xtakechars(atomChar, "atom")
+}
+
+func (p *parser) xmailbox() string {
+	s := p.xastring()
+	// UTF-7 is deprecated in IMAP4rev2. IMAP4rev1 does not fully forbid
+	// UTF-8 returned in mailbox names. We'll do our best by attempting to
+	// decode utf-7. But if that doesn't work, we'll just use the original
+	// string.
+	// ../rfc/3501:964
+	if !p.conn.enabled[capIMAP4rev2] {
+		ns, err := utf7decode(s)
+		if err != nil {
+			p.conn.log.Infox("decoding utf7 or mailbox name", err, mlog.Field("name", s))
+		} else {
+			s = ns
+		}
+	}
+	return s
+}
+
+// ../rfc/9051:6605
+func (p *parser) xlistMailbox() string {
+	if p.hasPrefix(`"`) || p.hasPrefix("{") {
+		return p.xstring()
+	}
+	return p.xtakechars(atomChar+listWildcards+respSpecials, "list-char")
+}
+
+// ../rfc/9051:6707 ../rfc/9051:6848 ../rfc/5258:1095 ../rfc/5258:1169 ../rfc/5258:1196
+func (p *parser) xmboxOrPat() ([]string, bool) {
+	if !p.take("(") {
+		return []string{p.xlistMailbox()}, false
+	}
+	l := []string{p.xlistMailbox()}
+	for !p.take(")") {
+		p.xspace()
+		l = append(l, p.xlistMailbox())
+	}
+	return l, true
+}
+
+// ../rfc/9051:7056
+// RECENT only in ../rfc/3501:5047
+// APPENDLIMIT is from ../rfc/7889:252
+func (p *parser) xstatusAtt() string {
+	return p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT")
+}
+
+// ../rfc/9051:7133 ../rfc/9051:7034
+func (p *parser) xnumSet() (r numSet) {
+	defer p.context("numSet")()
+	if p.take("$") {
+		return numSet{searchResult: true}
+	}
+	r.ranges = append(r.ranges, p.xnumRange())
+	for p.take(",") {
+		r.ranges = append(r.ranges, p.xnumRange())
+	}
+	return r
+}
+
+// parse numRange, which can be just a setNumber.
+func (p *parser) xnumRange() (r numRange) {
+	if p.take("*") {
+		r.first.star = true
+	} else {
+		r.first.number = p.xnznumber()
+	}
+	if p.take(":") {
+		r.last = &setNumber{}
+		if p.take("*") {
+			r.last.star = true
+		} else {
+			r.last.number = p.xnznumber()
+		}
+	}
+	return
+}
+
+// ../rfc/9051:6989 ../rfc/3501:4977
+func (p *parser) xsectionMsgtext() (r *sectionMsgtext) {
+	defer p.context("sectionMsgtext")()
+	msgtextWords := []string{"HEADER.FIELDS.NOT", "HEADER.FIELDS", "HEADER", "TEXT"}
+	w := p.xtakelist(msgtextWords...)
+	r = &sectionMsgtext{s: w}
+	if strings.HasPrefix(w, "HEADER.FIELDS") {
+		p.xspace()
+		p.xtake("(")
+		r.headers = append(r.headers, textproto.CanonicalMIMEHeaderKey(p.xastring()))
+		for {
+			if p.take(")") {
+				break
+			}
+			p.xspace()
+			r.headers = append(r.headers, textproto.CanonicalMIMEHeaderKey(p.xastring()))
+		}
+	}
+	return
+}
+
+// ../rfc/9051:6999 ../rfc/3501:4991
+func (p *parser) xsectionSpec() (r *sectionSpec) {
+	defer p.context("parseSectionSpec")()
+
+	n, ok := p.nznumber()
+	if !ok {
+		return &sectionSpec{msgtext: p.xsectionMsgtext()}
+	}
+	defer p.context("part...")()
+	pt := &sectionPart{}
+	pt.part = append(pt.part, n)
+	for {
+		if !p.take(".") {
+			break
+		}
+		if n, ok := p.nznumber(); ok {
+			pt.part = append(pt.part, n)
+			continue
+		}
+		if p.take("MIME") {
+			pt.text = &sectionText{mime: true}
+			break
+		}
+		pt.text = &sectionText{msgtext: p.xsectionMsgtext()}
+		break
+	}
+	return &sectionSpec{part: pt}
+}
+
+// ../rfc/9051:6985 ../rfc/3501:4975
+func (p *parser) xsection() *sectionSpec {
+	defer p.context("parseSection")()
+	p.xtake("[")
+	if p.take("]") {
+		return &sectionSpec{}
+	}
+	r := p.xsectionSpec()
+	p.xtake("]")
+	return r
+}
+
+// ../rfc/9051:6841
+func (p *parser) xpartial() *partial {
+	p.xtake("<")
+	offset := p.xnumber()
+	p.xtake(".")
+	count := p.xnznumber()
+	p.xtake(">")
+	return &partial{offset, count}
+}
+
+// ../rfc/9051:6987
+func (p *parser) xsectionBinary() (r []uint32) {
+	p.xtake("[")
+	if p.take("]") {
+		return nil
+	}
+	r = append(r, p.xnznumber())
+	for {
+		if !p.take(".") {
+			break
+		}
+		r = append(r, p.xnznumber())
+	}
+	p.xtake("]")
+	return r
+}
+
+// ../rfc/9051:6557 ../rfc/3501:4751
+func (p *parser) xfetchAtt() (r fetchAtt) {
+	defer p.context("fetchAtt")()
+	words := []string{
+		"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
+		"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
+	}
+	f := p.xtakelist(words...)
+	r.peek = strings.HasSuffix(f, ".PEEK")
+	r.field = strings.TrimSuffix(f, ".PEEK")
+
+	switch r.field {
+	case "BODY":
+		if p.hasPrefix("[") {
+			r.section = p.xsection()
+			if p.hasPrefix("<") {
+				r.partial = p.xpartial()
+			}
+		}
+	case "BINARY":
+		r.sectionBinary = p.xsectionBinary()
+		if p.hasPrefix("<") {
+			r.partial = p.xpartial()
+		}
+	case "BINARY.SIZE":
+		r.sectionBinary = p.xsectionBinary()
+	}
+	return
+}
+
+// ../rfc/9051:6553 ../rfc/3501:4748
+func (p *parser) xfetchAtts() []fetchAtt {
+	defer p.context("fetchAtts")()
+
+	fields := func(l ...string) []fetchAtt {
+		r := make([]fetchAtt, len(l))
+		for i, s := range l {
+			r[i] = fetchAtt{field: s}
+		}
+		return r
+	}
+
+	if w, ok := p.takelist("ALL", "FAST", "FULL"); ok {
+		switch w {
+		case "ALL":
+			return fields("FLAGS", "INTERNALDATE", "RFC822.SIZE", "ENVELOPE")
+		case "FAST":
+			return fields("FLAGS", "INTERNALDATE", "RFC822.SIZE")
+		case "FULL":
+			return fields("FLAGS", "INTERNALDATE", "RFC822.SIZE", "ENVELOPE", "BODY")
+		}
+		panic("missing case")
+	}
+
+	if !p.hasPrefix("(") {
+		return []fetchAtt{p.xfetchAtt()}
+	}
+
+	l := []fetchAtt{}
+	p.xtake("(")
+	for {
+		l = append(l, p.xfetchAtt())
+		if !p.take(" ") {
+			break
+		}
+	}
+	p.xtake(")")
+	return l
+}
+
+func xint(p *parser, s string) int {
+	v, err := strconv.ParseInt(s, 10, 32)
+	if err != nil {
+		p.xerrorf("bad int %q: %v", s, err)
+	}
+	return int(v)
+}
+
+func (p *parser) digit() (string, bool) {
+	if p.empty() {
+		return "", false
+	}
+	c := p.orig[p.o]
+	if c < '0' || c > '9' {
+		return "", false
+	}
+	s := p.orig[p.o : p.o+1]
+	p.o++
+	return s, true
+}
+
+func (p *parser) xdigit() string {
+	s, ok := p.digit()
+	if !ok {
+		p.xerrorf("expected digit")
+	}
+	return s
+}
+
+// ../rfc/9051:6492 ../rfc/3501:4695
+func (p *parser) xdateDayFixed() int {
+	if p.take(" ") {
+		return xint(p, p.xdigit())
+	}
+	return xint(p, p.xdigit()+p.xdigit())
+}
+
+var months = []string{"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"}
+
+// ../rfc/9051:6495 ../rfc/3501:4698
+func (p *parser) xdateMonth() time.Month {
+	s := strings.ToLower(p.xtaken(3))
+	for i, m := range months {
+		if m == s {
+			return time.Month(1 + i)
+		}
+	}
+	p.xerrorf("unknown month %q", s)
+	return 0
+}
+
+// ../rfc/9051:7120 ../rfc/3501:5067
+func (p *parser) xtime() (int, int, int) {
+	h := xint(p, p.xtaken(2))
+	p.xtake(":")
+	m := xint(p, p.xtaken(2))
+	p.xtake(":")
+	s := xint(p, p.xtaken(2))
+	return h, m, s
+}
+
+// ../rfc/9051:7159 ../rfc/3501:5083
+func (p *parser) xzone() (string, int) {
+	sign := p.xtakelist("+", "-")
+	s := p.xtaken(4)
+	v := xint(p, s)
+	seconds := (v/100)*3600 + (v%100)*60
+	if sign[0] == '-' {
+		seconds = -seconds
+	}
+	return sign + s, seconds
+}
+
+// ../rfc/9051:6502 ../rfc/3501:4713
+func (p *parser) xdateTime() time.Time {
+	// DQUOTE date-day-fixed "-" date-month "-" date-year SP time SP zone DQUOTE
+	p.xtake(`"`)
+	day := p.xdateDayFixed()
+	p.xtake("-")
+	month := p.xdateMonth()
+	p.xtake("-")
+	year := xint(p, p.xtaken(4))
+	p.xspace()
+	hours, minutes, seconds := p.xtime()
+	p.xspace()
+	name, zoneSeconds := p.xzone()
+	p.xtake(`"`)
+	loc := time.FixedZone(name, zoneSeconds)
+	return time.Date(year, month, day, hours, minutes, seconds, 0, loc)
+}
+
+// ../rfc/9051:6655 ../rfc/7888:330 ../rfc/3501:4801
+func (p *parser) xliteralSize(maxSize int64, lit8 bool) (size int64, sync bool) {
+	// todo: enforce that we get non-binary when ~ isn't present?
+	if lit8 {
+		p.take("~")
+	}
+	p.xtake("{")
+	size = p.xnumber64()
+	if maxSize > 0 && size > maxSize {
+		// ../rfc/7888:249
+		line := fmt.Sprintf("* BYE [ALERT] Max literal size %d is larger than allowed %d in this context", size, maxSize)
+		panic(syntaxError{line, "TOOBIG", fmt.Errorf("literal too big")})
+	}
+
+	sync = !p.take("+")
+	p.xtake("}")
+	p.xempty()
+	return size, sync
+}
+
+var searchKeyWords = []string{
+	"ALL", "ANSWERED", "BCC",
+	"BEFORE", "BODY",
+	"CC", "DELETED", "FLAGGED",
+	"FROM", "KEYWORD",
+	"NEW", "OLD", "ON", "RECENT", "SEEN",
+	"SINCE", "SUBJECT",
+	"TEXT", "TO",
+	"UNANSWERED", "UNDELETED", "UNFLAGGED",
+	"UNKEYWORD", "UNSEEN",
+	"DRAFT", "HEADER",
+	"LARGER", "NOT",
+	"OR",
+	"SENTBEFORE", "SENTON",
+	"SENTSINCE", "SMALLER",
+	"UID", "UNDRAFT",
+}
+
+// ../rfc/9051:6923 ../rfc/3501:4957
+// differences: rfc 9051 removes NEW, OLD, RECENT and makes SMALLER and LARGER number64 instead of number.
+func (p *parser) xsearchKey() *searchKey {
+	if p.take("(") {
+		sk := p.xsearchKey()
+		l := []searchKey{*sk}
+		for !p.take(")") {
+			p.xspace()
+			l = append(l, *p.xsearchKey())
+		}
+		return &searchKey{searchKeys: l}
+	}
+
+	w, ok := p.takelist(searchKeyWords...)
+	if !ok {
+		seqs := p.xnumSet()
+		return &searchKey{seqSet: &seqs}
+	}
+
+	sk := &searchKey{op: w}
+	switch sk.op {
+	case "ALL":
+	case "ANSWERED":
+	case "BCC":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "BEFORE":
+		p.xspace()
+		sk.date = p.xdate()
+	case "BODY":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "CC":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "DELETED":
+	case "FLAGGED":
+	case "FROM":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "KEYWORD":
+		p.xspace()
+		sk.atom = p.xatom()
+	case "NEW":
+	case "OLD":
+	case "ON":
+		p.xspace()
+		sk.date = p.xdate()
+	case "RECENT":
+	case "SEEN":
+	case "SINCE":
+		p.xspace()
+		sk.date = p.xdate()
+	case "SUBJECT":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "TEXT":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "TO":
+		p.xspace()
+		sk.astring = p.xastring()
+	case "UNANSWERED":
+	case "UNDELETED":
+	case "UNFLAGGED":
+	case "UNKEYWORD":
+		p.xspace()
+		sk.atom = p.xatom()
+	case "UNSEEN":
+	case "DRAFT":
+	case "HEADER":
+		p.xspace()
+		sk.headerField = p.xastring()
+		p.xspace()
+		sk.astring = p.xastring()
+	case "LARGER":
+		p.xspace()
+		sk.number = p.xnumber64()
+	case "NOT":
+		p.xspace()
+		sk.searchKey = p.xsearchKey()
+	case "OR":
+		p.xspace()
+		sk.searchKey = p.xsearchKey()
+		p.xspace()
+		sk.searchKey2 = p.xsearchKey()
+	case "SENTBEFORE":
+		p.xspace()
+		sk.date = p.xdate()
+	case "SENTON":
+		p.xspace()
+		sk.date = p.xdate()
+	case "SENTSINCE":
+		p.xspace()
+		sk.date = p.xdate()
+	case "SMALLER":
+		p.xspace()
+		sk.number = p.xnumber64()
+	case "UID":
+		p.xspace()
+		sk.uidSet = p.xnumSet()
+	case "UNDRAFT":
+	default:
+		p.xerrorf("missing case for op %q", sk.op)
+	}
+	return sk
+}
+
+// ../rfc/9051:6489 ../rfc/3501:4692
+func (p *parser) xdateDay() int {
+	d := p.xdigit()
+	if s, ok := p.digit(); ok {
+		d += s
+	}
+	return xint(p, d)
+}
+
+// ../rfc/9051:6487 ../rfc/3501:4690
+func (p *parser) xdate() time.Time {
+	dquote := p.take(`"`)
+	day := p.xdateDay()
+	p.xtake("-")
+	mon := p.xdateMonth()
+	p.xtake("-")
+	year := xint(p, p.xtaken(4))
+	if dquote {
+		p.take(`"`)
+	}
+	return time.Date(year, mon, day, 0, 0, 0, 0, time.UTC)
+}
+
+// ../rfc/9051:7090 ../rfc/4466:716
+func (p *parser) xtaggedExtLabel() string {
+	return p.xtake1fn(func(i int, c rune) bool {
+		return c >= 'A' && c <= 'Z' || c == '-' || c == '_' || c == '.' || i > 0 && (c >= '0' && c <= '9' || c == ':')
+	})
+}
+
+// no return value since we don't currently use the value.
+// ../rfc/9051:7111 ../rfc/4466:749
+func (p *parser) xtaggedExtVal() {
+	if p.take("(") {
+		if p.take(")") {
+			return
+		}
+		p.xtaggedExtComp()
+		p.xtake(")")
+	} else {
+		p.xtaggedExtSimple()
+	}
+}
+
+// ../rfc/9051:7109 ../rfc/4466:747
+func (p *parser) xtaggedExtSimple() {
+	s := p.digits()
+	if s == "" {
+		p.xnumSet()
+	}
+
+	// This can be a number64, or the start of a sequence-set. A sequence-set can also
+	// start with a number, but only an uint32. After the number we'll try to continue
+	// parsing as a sequence-set.
+	_, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		p.xerrorf("parsing int: %v", err)
+	}
+
+	if p.take(":") {
+		if !p.take("*") {
+			p.xnznumber()
+		}
+	}
+	for p.take(",") {
+		p.xnumRange()
+	}
+}
+
+// ../rfc/9051:7111 ../rfc/4466:735
+func (p *parser) xtaggedExtComp() {
+	if p.take("(") {
+		p.xtaggedExtComp()
+		p.xtake(")")
+		return
+	}
+	p.xastring()
+	for p.space() {
+		p.xtaggedExtComp()
+	}
+}

imapserver/prefixconn.go

@@ -0,0 +1,28 @@
+package imapserver
+
+import (
+	"net"
+)
+
+// prefixConn is a net.Conn with a buffer from which the first reads are satisfied.
+// used for STARTTLS where already did a buffered read of initial TLS data.
+type prefixConn struct {
+	prefix []byte
+	net.Conn
+}
+
+func (c *prefixConn) Read(buf []byte) (int, error) {
+	if len(c.prefix) > 0 {
+		n := len(buf)
+		if n > len(c.prefix) {
+			n = len(c.prefix)
+		}
+		copy(buf[:n], c.prefix[:n])
+		c.prefix = c.prefix[n:]
+		if len(c.prefix) == 0 {
+			c.prefix = nil
+		}
+		return n, nil
+	}
+	return c.Conn.Read(buf)
+}

imapserver/protocol.go

@@ -0,0 +1,186 @@
+package imapserver
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/mjl-/mox/store"
+)
+
+type numSet struct {
+	searchResult bool // "$"
+	ranges       []numRange
+}
+
+// containsSeq returns whether seq is in the numSet, given uids and (saved) searchResult.
+// uids and searchResult must be sorted. searchResult can have uids that are no longer in uids.
+func (ss numSet) containsSeq(seq msgseq, uids []store.UID, searchResult []store.UID) bool {
+	if len(uids) == 0 {
+		return false
+	}
+	if ss.searchResult {
+		uid := uids[int(seq)-1]
+		return uidSearch(searchResult, uid) > 0 && uidSearch(uids, uid) > 0
+	}
+	for _, r := range ss.ranges {
+		first := r.first.number
+		if r.first.star {
+			first = 1
+		}
+		last := first
+		if r.last != nil {
+			last = r.last.number
+			if r.last.star {
+				last = uint32(len(uids))
+			}
+		}
+		if last > uint32(len(uids)) {
+			last = uint32(len(uids))
+		}
+		if uint32(seq) >= first && uint32(seq) <= last {
+			return true
+		}
+	}
+	return false
+}
+
+func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []store.UID) bool {
+	if len(uids) == 0 {
+		return false
+	}
+	if ss.searchResult {
+		return uidSearch(searchResult, uid) > 0 && uidSearch(uids, uid) > 0
+	}
+	for _, r := range ss.ranges {
+		first := store.UID(r.first.number)
+		if r.first.star {
+			first = uids[0]
+		}
+		last := first
+		// Num in <num>:* can be larger than last, but it still matches the last...
+		// Similar for *:<num>. ../rfc/9051:4814
+		if r.last != nil {
+			last = store.UID(r.last.number)
+			if r.last.star {
+				last = uids[len(uids)-1]
+				if last > first {
+					first = last
+				}
+			} else if r.first.star && last < first {
+				last = first
+			}
+		}
+		if uid < first || uid > last {
+			continue
+		}
+		if uidSearch(uids, uid) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func (ss numSet) String() string {
+	if ss.searchResult {
+		return "$"
+	}
+	s := ""
+	for _, r := range ss.ranges {
+		if s != "" {
+			s += ","
+		}
+		if r.first.star {
+			s += "*"
+		} else {
+			s += fmt.Sprintf("%d", r.first.number)
+		}
+		if r.last == nil {
+			if r.first.star {
+				panic("invalid numSet range first star without last")
+			}
+			continue
+		}
+		s += ":"
+		if r.last.star {
+			s += "*"
+		} else {
+			s += fmt.Sprintf("%d", r.last.number)
+		}
+	}
+	return s
+}
+
+type setNumber struct {
+	number uint32
+	star   bool
+}
+
+type numRange struct {
+	first setNumber
+	last  *setNumber // if nil, this numRange is just a setNumber in "first" and first.star will be false
+}
+
+type partial struct {
+	offset uint32
+	count  uint32
+}
+
+type sectionPart struct {
+	part []uint32
+	text *sectionText
+}
+
+type sectionText struct {
+	mime    bool // if "MIME"
+	msgtext *sectionMsgtext
+}
+
+// a non-nil *sectionSpec with nil msgtext & nil part means there were []'s, but nothing inside. e.g. "BODY[]".
+type sectionSpec struct {
+	msgtext *sectionMsgtext
+	part    *sectionPart
+}
+
+type sectionMsgtext struct {
+	s       string   // "HEADER", "HEADER.FIELDS", "HEADER.FIELDS.NOT", "TEXT"
+	headers []string // for "HEADER.FIELDS"*
+}
+
+type fetchAtt struct {
+	field         string // uppercase, eg "ENVELOPE", "BODY". ".PEEK" is removed.
+	peek          bool
+	section       *sectionSpec
+	sectionBinary []uint32
+	partial       *partial
+}
+
+type searchKey struct {
+	// Only one of searchKeys, seqSet and op can be non-nil/non-empty.
+	searchKeys  []searchKey // In case of nested/multiple keys. Also for the top-level command.
+	seqSet      *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
+	op          string      // Determines which of the fields below are set.
+	headerField string
+	astring     string
+	date        time.Time
+	atom        string
+	number      int64
+	searchKey   *searchKey
+	searchKey2  *searchKey
+	uidSet      numSet
+}
+
+func compactUIDSet(l []store.UID) (r numSet) {
+	for len(l) > 0 {
+		e := 1
+		for ; e < len(l) && l[e] == l[e-1]+1; e++ {
+		}
+		first := setNumber{number: uint32(l[0])}
+		var last *setNumber
+		if e > 1 {
+			last = &setNumber{number: uint32(l[e-1])}
+		}
+		r.ranges = append(r.ranges, numRange{first, last})
+		l = l[e:]
+	}
+	return
+}

imapserver/protocol_test.go

@@ -0,0 +1,61 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/store"
+)
+
+func TestNumSetContains(t *testing.T) {
+	num := func(v uint32) *setNumber {
+		return &setNumber{v, false}
+	}
+	star := &setNumber{star: true}
+
+	check := func(v bool) {
+		t.Helper()
+		if !v {
+			t.Fatalf("bad")
+		}
+	}
+
+	ss0 := numSet{true, nil} // "$"
+	check(ss0.containsSeq(1, []store.UID{2}, []store.UID{2}))
+	check(!ss0.containsSeq(1, []store.UID{2}, []store.UID{}))
+
+	check(ss0.containsUID(1, []store.UID{1}, []store.UID{1}))
+	check(ss0.containsUID(2, []store.UID{1, 2, 3}, []store.UID{2}))
+	check(!ss0.containsUID(2, []store.UID{1, 2, 3}, []store.UID{}))
+	check(!ss0.containsUID(2, []store.UID{}, []store.UID{2}))
+
+	ss1 := numSet{false, []numRange{{*num(1), nil}}} // Single number 1.
+	check(ss1.containsSeq(1, []store.UID{2}, nil))
+	check(!ss1.containsSeq(2, []store.UID{1, 2}, nil))
+
+	check(ss1.containsUID(1, []store.UID{1}, nil))
+	check(ss1.containsSeq(1, []store.UID{2}, nil))
+	check(!ss1.containsSeq(2, []store.UID{1, 2}, nil))
+
+	// 2:*
+	ss2 := numSet{false, []numRange{{*num(2), star}}}
+	check(!ss2.containsSeq(1, []store.UID{2}, nil))
+	check(ss2.containsSeq(2, []store.UID{4, 5}, nil))
+	check(ss2.containsSeq(3, []store.UID{4, 5, 6}, nil))
+
+	check(ss2.containsUID(2, []store.UID{2}, nil))
+	check(ss2.containsUID(3, []store.UID{1, 2, 3}, nil))
+	check(ss2.containsUID(2, []store.UID{2}, nil))
+	check(!ss2.containsUID(2, []store.UID{4, 5}, nil))
+	check(!ss2.containsUID(2, []store.UID{1}, nil))
+
+	// *:2
+	ss3 := numSet{false, []numRange{{*star, num(2)}}}
+	check(ss3.containsSeq(1, []store.UID{2}, nil))
+	check(ss3.containsSeq(2, []store.UID{4, 5}, nil))
+	check(!ss3.containsSeq(3, []store.UID{1, 2, 3}, nil))
+
+	check(ss3.containsUID(1, []store.UID{1}, nil))
+	check(ss3.containsUID(2, []store.UID{1, 2, 3}, nil))
+	check(!ss3.containsUID(1, []store.UID{2, 3}, nil))
+	check(!ss3.containsUID(3, []store.UID{1, 2, 3}, nil))
+}

imapserver/rename_test.go

@@ -0,0 +1,81 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+// todo: check that UIDValidity is indeed updated properly.
+func TestRename(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Login("mjl@mox.example", "testtest")
+
+	tc.transactf("bad", "rename")      // Missing parameters.
+	tc.transactf("bad", "rename x")    // Missing destination.
+	tc.transactf("bad", "rename x y ") // Leftover data.
+
+	tc.transactf("no", "rename doesnotexist newbox") // Does not exist.
+	tc.xcode("NONEXISTENT")                          // ../rfc/9051:5140
+	tc.transactf("no", `rename "Sent" "Trash"`)      // Already exists.
+	tc.xcode("ALREADYEXISTS")
+
+	tc.client.Create("x")
+	tc.client.Subscribe("sub")
+	tc.client.Create("a/b/c")
+	tc.client.Subscribe("x/y/c") // For later rename, but not affected by rename of x.
+	tc2.transactf("ok", "noop")  // Drain.
+
+	tc.transactf("ok", "rename x y")
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "y", OldName: "x"})
+
+	// Rename to a mailbox that only exists in database as subscribed.
+	tc.transactf("ok", "rename y sub")
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "sub", OldName: "y"})
+
+	// Cannot rename a child to a parent. It already exists.
+	tc.transactf("no", "rename a/b/c a/b")
+	tc.xcode("ALREADYEXISTS")
+	tc.transactf("no", "rename a/b a")
+	tc.xcode("ALREADYEXISTS")
+
+	tc2.transactf("ok", "noop")          // Drain.
+	tc.transactf("ok", "rename a/b x/y") // This will cause new parent "x" to be created, and a/b and a/b/c to be renamed.
+	tc2.transactf("ok", "noop")
+	tc2.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "x"}, imapclient.UntaggedList{Separator: '/', Mailbox: "x/y", OldName: "a/b"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "x/y/c", OldName: "a/b/c"})
+
+	tc.client.Create("k/l")
+	tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
+	tc.transactf("ok", `list "" "k*" return (subscribed)`)
+	tc.xuntagged(imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{`\Subscribed`}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})
+
+	// Similar, but with missing parent not subscribed.
+	tc.transactf("ok", "rename k/l/m k/ll")
+	tc.transactf("ok", "delete k/l")
+	tc.transactf("ok", "rename k/ll k/l") // Restored to previous mailboxes now.
+	tc.client.Unsubscribe("k")
+	tc.transactf("ok", "rename k/l k/l/m") // With "l" renamed, a new "k" will be created.
+	tc.transactf("ok", `list "" "k*" return (subscribed)`)
+	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "k"}, imapclient.UntaggedList{Flags: []string{"\\Subscribed"}, Separator: '/', Mailbox: "k/l"}, imapclient.UntaggedList{Separator: '/', Mailbox: "k/l/m"})
+
+	// Renaming inbox keeps inbox in existence and does not rename children.
+	tc.transactf("ok", "create inbox/a")
+	tc.transactf("ok", "rename inbox minbox")
+	tc.transactf("ok", `list "" (inbox inbox/a minbox)`)
+	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox/a"}, imapclient.UntaggedList{Separator: '/', Mailbox: "minbox"})
+
+	// Renaming to new hiearchy that does not have any subscribes.
+	tc.transactf("ok", "rename minbox w/w")
+	tc.transactf("ok", `list "" "w*"`)
+	tc.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "w"}, imapclient.UntaggedList{Separator: '/', Mailbox: "w/w"})
+
+	// todo: test create+delete+rename of/to a name results in a higher uidvalidity.
+}

imapserver/search.go

@@ -0,0 +1,463 @@
+package imapserver
+
+import (
+	"fmt"
+	"io"
+	"net/textproto"
+	"strings"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/store"
+)
+
+// Search returns messages matching criteria specified in parameters.
+//
+// State: Selected
+func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
+	// Command: ../rfc/9051:3716 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723
+	// Examples: ../rfc/9051:3986 ../rfc/4731:153 ../rfc/3501:2975
+	// Syntax: ../rfc/9051:6918 ../rfc/4466:611 ../rfc/3501:4954
+
+	// We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2.
+	var eargs map[string]bool // Options except SAVE. Nil means old-style SEARCH response.
+	var save bool             // For SAVE option. Kept separately for easier handling of MIN/MAX later.
+
+	// IMAP4rev2 always returns ESEARCH, even with absent RETURN.
+	if c.enabled[capIMAP4rev2] {
+		eargs = map[string]bool{}
+	}
+	// ../rfc/9051:6967
+	if p.take(" RETURN (") {
+		eargs = map[string]bool{}
+
+		for !p.take(")") {
+			if len(eargs) > 0 || save {
+				p.xspace()
+			}
+			if w, ok := p.takelist("MIN", "MAX", "ALL", "COUNT", "SAVE"); ok {
+				if w == "SAVE" {
+					save = true
+				} else {
+					eargs[w] = true
+				}
+			} else {
+				// ../rfc/4466:378 ../rfc/9051:3745
+				xsyntaxErrorf("ESEARCH result option %q not supported", w)
+			}
+		}
+	}
+	// ../rfc/4731:149 ../rfc/9051:3737
+	if eargs != nil && len(eargs) == 0 && !save {
+		eargs["ALL"] = true
+	}
+
+	// If UTF8=ACCEPT is enabled, we should not accept any charset. We are a bit more
+	// relaxed (reasonable?) and still allow US-ASCII and UTF-8. ../rfc/6855:198
+	if p.take(" CHARSET ") {
+		charset := strings.ToUpper(p.xastring())
+		if charset != "US-ASCII" && charset != "UTF-8" {
+			// ../rfc/3501:2771 ../rfc/9051:3836
+			xusercodeErrorf("BADCHARSET", "only US-ASCII and UTF-8 supported")
+		}
+	}
+	p.xspace()
+	sk := &searchKey{
+		searchKeys: []searchKey{*p.xsearchKey()},
+	}
+	for !p.empty() {
+		p.xspace()
+		sk.searchKeys = append(sk.searchKeys, *p.xsearchKey())
+	}
+
+	// Even in case of error, we ensure search result is changed.
+	if save {
+		c.searchResult = []store.UID{}
+	}
+
+	// Note: we only hold the account rlock for verifying the mailbox at the start.
+	c.account.RLock()
+	runlock := c.account.RUnlock
+	// Note: in a defer because we replace it below.
+	defer func() {
+		runlock()
+	}()
+
+	// If we only have a MIN and/or MAX, we can stop processing as soon as we
+	// have those matches.
+	var min, max int
+	if eargs["MIN"] {
+		min = 1
+	}
+	if eargs["MAX"] {
+		max = 1
+	}
+
+	var expungeIssued bool
+
+	var uids []store.UID
+	c.xdbread(func(tx *bstore.Tx) {
+		c.xmailboxID(tx, c.mailboxID) // Validate.
+		runlock()
+		runlock = func() {}
+
+		// Normal forward search when we don't have MAX only.
+		var lastIndex = -1
+		if eargs == nil || max == 0 || len(eargs) != 1 {
+			for i, uid := range c.uids {
+				lastIndex = i
+				if c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued) {
+					uids = append(uids, uid)
+					if min == 1 && min+max == len(eargs) {
+						break
+					}
+				}
+			}
+		}
+		// And reverse search for MAX if we have only MAX or MAX combined with MIN.
+		if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) {
+			for i := len(c.uids) - 1; i > lastIndex; i-- {
+				if c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued) {
+					uids = append(uids, c.uids[i])
+					break
+				}
+			}
+		}
+	})
+
+	if eargs == nil {
+		// Old-style SEARCH response. We must spell out each number. So we may be splitting
+		// into multiple responses. ../rfc/9051:6809 ../rfc/3501:4833
+		for len(uids) > 0 {
+			n := len(uids)
+			if n > 100 {
+				n = 100
+			}
+			s := ""
+			for _, v := range uids[:n] {
+				if !isUID {
+					v = store.UID(c.xsequence(v))
+				}
+				s += " " + fmt.Sprintf("%d", v)
+			}
+			uids = uids[n:]
+			c.bwritelinef("* SEARCH%s", s)
+		}
+	} else {
+		// New-style ESEARCH response. ../rfc/9051:6546 ../rfc/4466:522
+
+		if save {
+			// ../rfc/9051:3784 ../rfc/5182:13
+			c.searchResult = uids
+			if sanityChecks {
+				checkUIDs(c.searchResult)
+			}
+		}
+
+		// No untagged ESEARCH response if nothing was requested. ../rfc/9051:4160
+		if len(eargs) > 0 {
+			resp := fmt.Sprintf("* ESEARCH (TAG %s)", tag)
+			if isUID {
+				resp += " UID"
+			}
+
+			// NOTE: we are converting UIDs to msgseq in the uids slice (if needed) while
+			// keeping the "uids" name!
+			if !isUID {
+				// If searchResult is hanging on to the slice, we need to work on a copy.
+				if save {
+					nuids := make([]store.UID, len(uids))
+					copy(nuids, uids)
+					uids = nuids
+				}
+				for i, uid := range uids {
+					uids[i] = store.UID(c.xsequence(uid))
+				}
+			}
+
+			// If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758
+			if eargs["MIN"] && len(uids) > 0 {
+				resp += fmt.Sprintf(" MIN %d", uids[0])
+			}
+			if eargs["MAX"] && len(uids) > 0 {
+				resp += fmt.Sprintf(" MAX %d", uids[len(uids)-1])
+			}
+			if eargs["COUNT"] {
+				resp += fmt.Sprintf(" COUNT %d", len(uids))
+			}
+			if eargs["ALL"] && len(uids) > 0 {
+				resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String())
+			}
+			c.bwritelinef("%s", resp)
+		}
+	}
+	if expungeIssued {
+		// ../rfc/9051:5102
+		c.writeresultf("%s OK [EXPUNGEISSUED] done", tag)
+	} else {
+		c.ok(tag, cmd)
+	}
+}
+
+type search struct {
+	c             *conn
+	tx            *bstore.Tx
+	seq           msgseq
+	uid           store.UID
+	mr            *store.MsgReader
+	m             store.Message
+	p             *message.Part
+	expungeIssued *bool
+}
+
+func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) bool {
+	s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued}
+	defer func() {
+		if s.mr != nil {
+			err := s.mr.Close()
+			c.xsanity(err, "closing messagereader")
+			s.mr = nil
+		}
+	}()
+	return s.match(sk)
+}
+
+func (s *search) match(sk searchKey) bool {
+	c := s.c
+
+	if sk.searchKeys != nil {
+		for _, ssk := range sk.searchKeys {
+			if !s.match(ssk) {
+				return false
+			}
+		}
+		return true
+	} else if sk.seqSet != nil {
+		return sk.seqSet.containsSeq(s.seq, c.uids, c.searchResult)
+	}
+
+	filterHeader := func(field, value string) bool {
+		lower := strings.ToLower(value)
+		h, err := s.p.Header()
+		if err != nil {
+			c.log.Debugx("parsing message header", err, mlog.Field("uid", s.uid))
+			return false
+		}
+		for _, v := range h.Values(field) {
+			if strings.Contains(strings.ToLower(v), lower) {
+				return true
+			}
+		}
+		return false
+	}
+
+	// We handle ops by groups that need increasing details about the message.
+
+	switch sk.op {
+	case "ALL":
+		return true
+	case "NEW":
+		// We do not implement the RECENT flag, so messages cannot be NEW.
+		return false
+	case "OLD":
+		// We treat all messages as non-recent, so this means all messages.
+		return true
+	case "RECENT":
+		// We do not implement the RECENT flag. All messages are not recent.
+		return false
+	case "NOT":
+		return !s.match(*sk.searchKey)
+	case "OR":
+		return s.match(*sk.searchKey) || s.match(*sk.searchKey2)
+	case "UID":
+		return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult)
+	}
+
+	// Parsed message.
+	if s.mr == nil {
+		q := bstore.QueryTx[store.Message](s.tx)
+		q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: s.uid})
+		m, err := q.Get()
+		if err == bstore.ErrAbsent {
+			// ../rfc/2180:607
+			*s.expungeIssued = true
+			return false
+		}
+		xcheckf(err, "get message")
+		s.m = m
+
+		// Closed by searchMatch after all (recursive) search.match calls are finished.
+		s.mr = c.account.MessageReader(m)
+
+		if m.ParsedBuf == nil {
+			c.log.Error("missing parsed message")
+		} else {
+			p, err := m.LoadPart(s.mr)
+			xcheckf(err, "load parsed message")
+			s.p = &p
+		}
+	}
+
+	// Parsed message, basic info.
+	switch sk.op {
+	case "ANSWERED":
+		return s.m.Answered
+	case "DELETED":
+		return s.m.Deleted
+	case "FLAGGED":
+		return s.m.Flagged
+	case "KEYWORD":
+		switch sk.atom {
+		case "$Forwarded":
+			return s.m.Forwarded
+		case "$Junk":
+			return s.m.Junk
+		case "$NotJunk":
+			return s.m.Notjunk
+		case "$Phishing":
+			return s.m.Phishing
+		case "$MDNSent":
+			return s.m.MDNSent
+		default:
+			c.log.Info("search with unknown keyword", mlog.Field("keyword", sk.atom))
+			return false
+		}
+	case "SEEN":
+		return s.m.Seen
+	case "UNANSWERED":
+		return !s.m.Answered
+	case "UNDELETED":
+		return !s.m.Deleted
+	case "UNFLAGGED":
+		return !s.m.Flagged
+	case "UNKEYWORD":
+		switch sk.atom {
+		case "$Forwarded":
+			return !s.m.Forwarded
+		case "$Junk":
+			return !s.m.Junk
+		case "$NotJunk":
+			return !s.m.Notjunk
+		case "$Phishing":
+			return !s.m.Phishing
+		case "$MDNSent":
+			return !s.m.MDNSent
+		default:
+			c.log.Info("search with unknown keyword", mlog.Field("keyword", sk.atom))
+			return false
+		}
+	case "UNSEEN":
+		return !s.m.Seen
+	case "DRAFT":
+		return s.m.Draft
+	case "UNDRAFT":
+		return !s.m.Draft
+	case "BEFORE", "ON", "SINCE":
+		skdt := sk.date.Format("2006-01-02")
+		rdt := s.m.Received.Format("2006-01-02")
+		switch sk.op {
+		case "BEFORE":
+			return rdt < skdt
+		case "ON":
+			return rdt == skdt
+		case "SINCE":
+			return rdt >= skdt
+		}
+		panic("missing case")
+	case "LARGER":
+		return s.m.Size > sk.number
+	case "SMALLER":
+		return s.m.Size < sk.number
+	}
+
+	if s.p == nil {
+		c.log.Info("missing parsed message, not matching", mlog.Field("uid", s.uid))
+		return false
+	}
+
+	// Parsed message, more info.
+	switch sk.op {
+	case "BCC":
+		return filterHeader("Bcc", sk.astring)
+	case "BODY", "TEXT":
+		headerToo := sk.op == "TEXT"
+		lower := strings.ToLower(sk.astring)
+		return mailContains(c, s.uid, s.p, lower, headerToo)
+	case "CC":
+		return filterHeader("Cc", sk.astring)
+	case "FROM":
+		return filterHeader("From", sk.astring)
+	case "SUBJECT":
+		return filterHeader("Subject", sk.astring)
+	case "TO":
+		return filterHeader("To", sk.astring)
+	case "HEADER":
+		// ../rfc/9051:3895
+		lower := strings.ToLower(sk.astring)
+		h, err := s.p.Header()
+		if err != nil {
+			c.log.Errorx("parsing header for search", err, mlog.Field("uid", s.uid))
+			return false
+		}
+		k := textproto.CanonicalMIMEHeaderKey(sk.headerField)
+		for _, v := range h.Values(k) {
+			if lower == "" || strings.Contains(strings.ToLower(v), lower) {
+				return true
+			}
+		}
+		return false
+	case "SENTBEFORE", "SENTON", "SENTSINCE":
+		if s.p.Envelope == nil || s.p.Envelope.Date.IsZero() {
+			return false
+		}
+		dt := s.p.Envelope.Date.Format("2006-01-02")
+		skdt := sk.date.Format("2006-01-02")
+		switch sk.op {
+		case "SENTBEFORE":
+			return dt < skdt
+		case "SENTON":
+			return dt == skdt
+		case "SENTSINCE":
+			return dt > skdt
+		}
+		panic("missing case")
+	}
+	panic(serverError{fmt.Errorf("missing case for search key op %q", sk.op)})
+}
+
+// mailContains returns whether the mail message or part represented by p contains (case-insensitive) string lower.
+// The (decoded) text bodies are tested for a match.
+// If headerToo is set, the header part of the message is checked as well.
+func mailContains(c *conn, uid store.UID, p *message.Part, lower string, headerToo bool) bool {
+	if headerToo && mailContainsReader(c, uid, p.HeaderReader(), lower) {
+		return true
+	}
+
+	if len(p.Parts) == 0 {
+		if p.MediaType != "TEXT" {
+			// todo: for types we could try to find a library for parsing and search in there too
+			return false
+		}
+		// todo: for html and perhaps other types, we could try to parse as text and filter on the text.
+		return mailContainsReader(c, uid, p.Reader(), lower)
+	}
+	for _, pp := range p.Parts {
+		headerToo = pp.MediaType == "MESSAGE" && (pp.MediaSubType == "RFC822" || pp.MediaSubType == "GLOBAL")
+		if mailContains(c, uid, &pp, lower, headerToo) {
+			return true
+		}
+	}
+	return false
+}
+
+func mailContainsReader(c *conn, uid store.UID, r io.Reader, lower string) bool {
+	// todo: match as we read
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		c.log.Errorx("reading for search text match", err, mlog.Field("uid", uid))
+		return false
+	}
+	return strings.Contains(strings.ToLower(string(buf)), lower)
+}

imapserver/search_test.go

@@ -0,0 +1,345 @@
+package imapserver
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+var searchMsg = strings.ReplaceAll(`Date: Mon, 1 Jan 2022 10:00:00 +0100 (CEST)
+From: mjl <mjl@mox.example>
+Subject: mox
+To: mox <mox@mox.example>
+Cc: <xcc@mox.example>
+Bcc: <bcc@mox.example>
+Reply-To: <noreply@mox.example>
+Message-Id: <123@mox.example>
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary=x
+
+--x
+Content-Type: text/plain; charset=utf-8
+
+this is plain text.
+
+--x
+Content-Type: text/html; charset=utf-8
+
+this is html.
+
+--x--
+`, "\n", "\r\n")
+
+func (tc *testconn) xsearch(nums ...uint32) {
+	tc.t.Helper()
+
+	if len(nums) == 0 {
+		tc.xnountagged()
+		return
+	}
+	tc.xuntagged(imapclient.UntaggedSearch(nums))
+}
+
+func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) {
+	tc.t.Helper()
+
+	exp.Correlator = tc.client.LastTag
+	tc.xuntagged(exp)
+}
+
+func TestSearch(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+
+	// Add 5 and delete first 4 messages. So UIDs start at 5.
+	received := time.Date(2020, time.January, 1, 10, 0, 0, 0, time.UTC)
+	for i := 0; i < 5; i++ {
+		tc.client.Append("inbox", nil, &received, []byte(exampleMsg))
+	}
+	tc.client.StoreFlagsSet("1:4", true, `\Deleted`)
+	tc.client.Expunge()
+
+	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
+	tc.client.Append("inbox", nil, &received, []byte(searchMsg))
+
+	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
+	mostFlags := []string{
+		`\Deleted`,
+		`\Seen`,
+		`\Answered`,
+		`\Flagged`,
+		`\Draft`,
+		`$Forwarded`,
+		`$Junk`,
+		`$Notjunk`,
+		`$Phishing`,
+		`$MDNSent`,
+	}
+	tc.client.Append("inbox", mostFlags, &received, []byte(searchMsg))
+
+	// We now have sequence numbers 1,2,3 and UIDs 5,6,7.
+
+	tc.transactf("ok", "search all")
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", "uid search all")
+	tc.xsearch(5, 6, 7)
+
+	tc.transactf("ok", "search answered")
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search bcc "bcc@mox.example"`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", "search before 1-Jan-2038")
+	tc.xsearch(1, 2, 3)
+	tc.transactf("ok", "search before 1-Jan-2020")
+	tc.xsearch() // Before is about received, not date header of message.
+
+	tc.transactf("ok", `search body "Joe"`)
+	tc.xsearch(1)
+	tc.transactf("ok", `search body "this is plain text"`)
+	tc.xsearch(2, 3)
+	tc.transactf("ok", `search body "this is html"`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", `search cc "xcc@mox.example"`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", `search deleted`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search flagged`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search from "foobar@Blurdybloop.example"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search keyword $Forwarded`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search new`)
+	tc.xsearch() // New requires a message to be recent. We pretend all messages are not recent.
+
+	tc.transactf("ok", `search old`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search on 1-Jan-2022`)
+	tc.xsearch(2, 3)
+
+	tc.transactf("ok", `search recent`)
+	tc.xsearch()
+
+	tc.transactf("ok", `search seen`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search since 1-Jan-2020`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search subject "afternoon"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search text "Joe"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search to "mooch@owatagu.siam.edu.example"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search unanswered`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search undeleted`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unflagged`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unkeyword $Junk`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search unseen`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("ok", `search draft`)
+	tc.xsearch(3)
+
+	tc.transactf("ok", `search header "subject" "afternoon"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search larger 1`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search not text "mox"`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search or seen unseen`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search or unseen seen`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search sentbefore 8-Feb-1994`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search senton 7-Feb-1994`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search sentsince 6-Feb-1994`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search smaller 9999999`)
+	tc.xsearch(1, 2, 3)
+
+	tc.transactf("ok", `search uid 1`)
+	tc.xsearch()
+
+	tc.transactf("ok", `search uid 5`)
+	tc.xsearch(1)
+
+	tc.transactf("ok", `search undraft`)
+	tc.xsearch(1, 2)
+
+	tc.transactf("no", `search charset unknown text "mox"`)
+	tc.transactf("ok", `search charset us-ascii text "mox"`)
+	tc.xsearch(2, 3)
+	tc.transactf("ok", `search charset utf-8 text "mox"`)
+	tc.xsearch(2, 3)
+
+	// esearchall makes an UntaggedEsearch response with All set, for comparisons.
+	esearchall0 := func(ss string) imapclient.NumSet {
+		seqset := imapclient.NumSet{}
+		for _, rs := range strings.Split(ss, ",") {
+			t := strings.Split(rs, ":")
+			if len(t) > 2 {
+				panic("bad seqset")
+			}
+			var first uint32
+			var last *uint32
+			if t[0] != "*" {
+				v, err := strconv.ParseUint(t[0], 10, 32)
+				if err != nil {
+					panic("parse first")
+				}
+				first = uint32(v)
+			}
+			if len(t) == 2 {
+				if t[1] != "*" {
+					v, err := strconv.ParseUint(t[1], 10, 32)
+					if err != nil {
+						panic("parse last")
+					}
+					u := uint32(v)
+					last = &u
+				}
+			}
+			seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
+		}
+		return seqset
+	}
+
+	esearchall := func(ss string) imapclient.UntaggedEsearch {
+		return imapclient.UntaggedEsearch{All: esearchall0(ss)}
+	}
+
+	uintptr := func(v uint32) *uint32 {
+		return &v
+	}
+
+	// Do new-style ESEARCH requests with RETURN. We should get an ESEARCH response.
+	tc.transactf("ok", "search return () all")
+	tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
+
+	tc.transactf("ok", "search return (min max count all) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(3), All: esearchall0("1:3")})
+
+	tc.transactf("ok", "UID search return (min max count all) all")
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(3), All: esearchall0("5:7")})
+
+	tc.transactf("ok", "search return (min) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
+
+	tc.transactf("ok", "search return (min) 3")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 3})
+
+	tc.transactf("ok", "search return (min) NOT all")
+	tc.xesearch(imapclient.UntaggedEsearch{}) // Min not present if no match.
+
+	tc.transactf("ok", "search return (max) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Max: 3})
+
+	tc.transactf("ok", "search return (max) 1")
+	tc.xesearch(imapclient.UntaggedEsearch{Max: 1})
+
+	tc.transactf("ok", "search return (max) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{}) // Max not present if no match.
+
+	tc.transactf("ok", "search return (min max) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3})
+
+	tc.transactf("ok", "search return (min max) 1")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1})
+
+	tc.transactf("ok", "search return (min max) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{})
+
+	tc.transactf("ok", "search return (all) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{}) // All not present if no match.
+
+	tc.transactf("ok", "search return (min max all) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{})
+
+	tc.transactf("ok", "search return (min max all count) not all")
+	tc.xesearch(imapclient.UntaggedEsearch{Count: uintptr(0)})
+
+	tc.transactf("ok", "search return (min max count all) 1,3")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
+
+	tc.transactf("ok", "search return (min max count all) UID 5,7")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
+
+	tc.transactf("ok", "uid search return (min max count all) 1,3")
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
+
+	tc.transactf("ok", "uid search return (min max count all) UID 5,7")
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
+
+	tc.transactf("no", `search return () charset unknown text "mox"`)
+	tc.transactf("ok", `search return () charset us-ascii text "mox"`)
+	tc.xesearch(esearchall("2:3"))
+	tc.transactf("ok", `search return () charset utf-8 text "mox"`)
+	tc.xesearch(esearchall("2:3"))
+
+	tc.transactf("bad", `search return (unknown) all`)
+
+	tc.transactf("ok", "search return (save) 2")
+	tc.xnountagged() // ../rfc/9051:3800
+	tc.transactf("ok", "fetch $ (uid)")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6)}})
+
+	tc.transactf("ok", "search return (all) $")
+	tc.xesearch(esearchall("2"))
+
+	tc.transactf("ok", "search return (save) $")
+	tc.xnountagged()
+
+	tc.transactf("ok", "search return (save all) all")
+	tc.xesearch(esearchall("1:3"))
+
+	tc.transactf("ok", "search return (all save) all")
+	tc.xesearch(esearchall("1:3"))
+
+	tc.transactf("ok", "search return (min save) all")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
+	tc.transactf("ok", "fetch $ (uid)")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5)}})
+
+	// Do a seemingly old-style search command with IMAP4rev2 enabled. We'll still get ESEARCH responses.
+	tc.client.Enable("IMAP4rev2")
+	tc.transactf("ok", `search undraft`)
+	tc.xesearch(esearchall("1:2"))
+}

imapserver/selectexamine_test.go

@@ -0,0 +1,71 @@
+package imapserver
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestSelect(t *testing.T) {
+	testSelectExamine(t, false)
+}
+
+func TestExamine(t *testing.T) {
+	testSelectExamine(t, true)
+}
+
+// select and examine are pretty much the same. but examine opens readonly instead of readwrite.
+func testSelectExamine(t *testing.T, examine bool) {
+	defer mockUIDValidity()()
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	cmd := "select"
+	okcode := "READ-WRITE"
+	if examine {
+		cmd = "examine"
+		okcode = "READ-ONLY"
+	}
+
+	uclosed := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "CLOSED", More: "x"}}
+	flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent`, " ")
+	uflags := imapclient.UntaggedFlags(flags)
+	upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: flags}, More: "x"}}
+	urecent := imapclient.UntaggedRecent(0)
+	uexists0 := imapclient.UntaggedExists(0)
+	uexists1 := imapclient.UntaggedExists(1)
+	uuidval1 := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}}
+	uuidnext1 := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 1}, More: "x"}}
+	ulist := imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}
+	uunseen := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}}
+	uuidnext2 := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 2}, More: "x"}}
+
+	// Parameter required.
+	tc.transactf("bad", cmd)
+
+	// Mailbox does not exist.
+	tc.transactf("no", cmd+" bogus")
+
+	tc.transactf("ok", cmd+" inbox")
+	tc.xuntagged(uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist)
+	tc.xcode(okcode)
+
+	tc.transactf("ok", cmd+` "inbox"`)
+	tc.xuntagged(uclosed, uflags, upermflags, urecent, uexists0, uuidval1, uuidnext1, ulist)
+	tc.xcode(okcode)
+
+	// Append a message. It will be reported as UNSEEN.
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.transactf("ok", cmd+" inbox")
+	tc.xuntagged(uclosed, uflags, upermflags, urecent, uunseen, uexists1, uuidval1, uuidnext2, ulist)
+	tc.xcode(okcode)
+
+	// With imap4rev2, we no longer get untagged RECENT or untagged UNSEEN.
+	tc.client.Enable("imap4rev2")
+	tc.transactf("ok", cmd+" inbox")
+	tc.xuntagged(uclosed, uflags, upermflags, uexists1, uuidval1, uuidnext2, ulist)
+	tc.xcode(okcode)
+}

imapserver/server_test.go

@@ -0,0 +1,646 @@
+package imapserver
+
+import (
+	"context"
+	"crypto/ed25519"
+	cryptorand "crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/imapclient"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/store"
+)
+
+func init() {
+	sanityChecks = true
+}
+
+func tocrlf(s string) string {
+	return strings.ReplaceAll(s, "\n", "\r\n")
+}
+
+// From ../rfc/3501:2589
+var exampleMsg = tocrlf(`Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)
+From: Fred Foobar <foobar@Blurdybloop.example>
+Subject: afternoon meeting
+To: mooch@owatagu.siam.edu.example
+Message-Id: <B27397-0100000@Blurdybloop.example>
+MIME-Version: 1.0
+Content-Type: TEXT/PLAIN; CHARSET=US-ASCII
+
+Hello Joe, do you think we can meet at 3:30 tomorrow?
+
+`)
+
+/*
+From ../rfc/2049:801
+
+Message structure:
+
+Message - multipart/mixed
+Part 1 - no content-type
+Part 2 - text/plain
+Part 3 - multipart/parallel
+Part 3.1 - audio/basic (base64)
+Part 3.2 - image/jpeg (base64, empty)
+Part 4 - text/enriched
+Part 5 - message/rfc822
+Part 5.1 - text/plain (quoted-printable)
+*/
+var nestedMessage = tocrlf(`MIME-Version: 1.0
+From: Nathaniel Borenstein <nsb@nsb.fv.com>
+To: Ned Freed <ned@innosoft.com>
+Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT)
+Subject: A multipart example
+Content-Type: multipart/mixed;
+              boundary=unique-boundary-1
+
+This is the preamble area of a multipart message.
+Mail readers that understand multipart format
+should ignore this preamble.
+
+If you are reading this text, you might want to
+consider changing to a mail reader that understands
+how to properly display multipart messages.
+
+--unique-boundary-1
+
+  ... Some text appears here ...
+
+[Note that the blank between the boundary and the start
+ of the text in this part means no header fields were
+ given and this is text in the US-ASCII character set.
+ It could have been done with explicit typing as in the
+ next part.]
+
+--unique-boundary-1
+Content-type: text/plain; charset=US-ASCII
+
+This could have been part of the previous part, but
+illustrates explicit versus implicit typing of body
+parts.
+
+--unique-boundary-1
+Content-Type: multipart/parallel; boundary=unique-boundary-2
+
+--unique-boundary-2
+Content-Type: audio/basic
+Content-Transfer-Encoding: base64
+
+aGVsbG8NCndvcmxkDQo=
+
+--unique-boundary-2
+Content-Type: image/jpeg
+Content-Transfer-Encoding: base64
+
+
+--unique-boundary-2--
+
+--unique-boundary-1
+Content-type: text/enriched
+
+This is <bold><italic>enriched.</italic></bold>
+<smaller>as defined in RFC 1896</smaller>
+
+Isn't it
+<bigger><bigger>cool?</bigger></bigger>
+
+--unique-boundary-1
+Content-Type: message/rfc822
+
+From: info@mox.example
+To: mox <info@mox.example>
+Subject: (subject in US-ASCII)
+Content-Type: Text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: Quoted-printable
+
+  ... Additional text in ISO-8859-1 goes here ...
+
+--unique-boundary-1--
+`)
+
+func tcheck(t *testing.T, err error, msg string) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("%s: %s", msg, err)
+	}
+}
+
+func mockUIDValidity() func() {
+	orig := store.InitialUIDValidity
+	store.InitialUIDValidity = func() uint32 {
+		return 1
+	}
+	return func() {
+		store.InitialUIDValidity = orig
+	}
+}
+
+type testconn struct {
+	t          *testing.T
+	conn       net.Conn
+	client     *imapclient.Conn
+	done       chan struct{}
+	serverConn net.Conn
+
+	// Result of last command.
+	lastUntagged []imapclient.Untagged
+	lastResult   imapclient.Result
+	lastErr      error
+}
+
+func (tc *testconn) check(err error, msg string) {
+	tc.t.Helper()
+	if err != nil {
+		tc.t.Fatalf("%s: %s", msg, err)
+	}
+}
+
+func (tc *testconn) last(l []imapclient.Untagged, r imapclient.Result, err error) {
+	tc.lastUntagged = l
+	tc.lastResult = r
+	tc.lastErr = err
+}
+
+func (tc *testconn) xcode(s string) {
+	tc.t.Helper()
+	if tc.lastResult.Code != s {
+		tc.t.Fatalf("got last code %q, expected %q", tc.lastResult.Code, s)
+	}
+}
+
+func (tc *testconn) xcodeArg(v any) {
+	tc.t.Helper()
+	if !reflect.DeepEqual(tc.lastResult.CodeArg, v) {
+		tc.t.Fatalf("got last code argument %v, expected %v", tc.lastResult.CodeArg, v)
+	}
+}
+
+func (tc *testconn) xuntagged(exps ...any) {
+	tc.t.Helper()
+	last := append([]imapclient.Untagged{}, tc.lastUntagged...)
+next:
+	for ei, exp := range exps {
+		for i, l := range last {
+			if reflect.TypeOf(l) != reflect.TypeOf(exp) {
+				continue
+			}
+			if !reflect.DeepEqual(l, exp) {
+				tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", l, l, exp, exp)
+			}
+			copy(last[i:], last[i+1:])
+			last = last[:len(last)-1]
+			continue next
+		}
+		var next string
+		if len(tc.lastUntagged) > 0 {
+			next = fmt.Sprintf(", next %#v", tc.lastUntagged[0])
+		}
+		tc.t.Fatalf("did not find untagged response %#v %T (%d) in %v%s", exp, exp, ei, tc.lastUntagged, next)
+	}
+	if len(last) > 0 {
+		tc.t.Fatalf("leftover untagged responses %v", last)
+	}
+}
+
+func tuntagged(t *testing.T, got imapclient.Untagged, dst any) {
+	t.Helper()
+	gotv := reflect.ValueOf(got)
+	dstv := reflect.ValueOf(dst)
+	if gotv.Type() != dstv.Type().Elem() {
+		t.Fatalf("got %v, expected %v", gotv.Type(), dstv.Type().Elem())
+	}
+	dstv.Elem().Set(gotv)
+}
+
+func (tc *testconn) xnountagged() {
+	tc.t.Helper()
+	if len(tc.lastUntagged) != 0 {
+		tc.t.Fatalf("got %v untagged, expected 0", tc.lastUntagged)
+	}
+}
+
+func (tc *testconn) transactf(status, format string, args ...any) {
+	tc.t.Helper()
+	tc.cmdf("", format, args...)
+	tc.response(status)
+}
+
+func (tc *testconn) response(status string) {
+	tc.t.Helper()
+	tc.lastUntagged, tc.lastResult, tc.lastErr = tc.client.Response()
+	tcheck(tc.t, tc.lastErr, "read imap response")
+	if strings.ToUpper(status) != string(tc.lastResult.Status) {
+		tc.t.Fatalf("got status %q, expected %q", tc.lastResult.Status, status)
+	}
+}
+
+func (tc *testconn) cmdf(tag, format string, args ...any) {
+	tc.t.Helper()
+	err := tc.client.Commandf(tag, format, args...)
+	tcheck(tc.t, err, "writing imap command")
+}
+
+func (tc *testconn) readstatus(status string) {
+	tc.t.Helper()
+	tc.response(status)
+}
+
+func (tc *testconn) readprefixline(pre string) {
+	tc.t.Helper()
+	line, err := tc.client.Readline()
+	tcheck(tc.t, err, "read line")
+	if !strings.HasPrefix(line, pre) {
+		tc.t.Fatalf("expected prefix %q, got %q", pre, line)
+	}
+}
+
+func (tc *testconn) writelinef(format string, args ...any) {
+	tc.t.Helper()
+	err := tc.client.Writelinef(format, args...)
+	tcheck(tc.t, err, "write line")
+}
+
+// wait at most 1 second for server to quit.
+func (tc *testconn) waitDone() {
+	tc.t.Helper()
+	t := time.NewTimer(time.Second)
+	select {
+	case <-tc.done:
+		t.Stop()
+	case <-t.C:
+		tc.t.Fatalf("server not done within 1s")
+	}
+}
+
+func (tc *testconn) close() {
+	tc.client.Close()
+	tc.serverConn.Close()
+	tc.waitDone()
+}
+
+var connCounter int64
+
+func start(t *testing.T) *testconn {
+	return startArgs(t, true, false, true)
+}
+
+func startNoSwitchboard(t *testing.T) *testconn {
+	return startArgs(t, false, false, true)
+}
+
+func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn {
+	if first {
+		os.RemoveAll("../testdata/imap/data")
+	}
+	mox.Context = context.Background()
+	mox.ConfigStaticPath = "../testdata/imap/mox.conf"
+	mox.MustLoadConfig()
+	acc, err := store.OpenAccount("mjl")
+	tcheck(t, err, "open account")
+	if first {
+		err = acc.SetPassword("testtest")
+		tcheck(t, err, "set password")
+	}
+	err = acc.Close()
+	tcheck(t, err, "close account")
+	var switchDone chan struct{}
+	if first {
+		switchDone = store.Switchboard()
+	} else {
+		switchDone = make(chan struct{}) // Dummy, that can be closed.
+	}
+
+	serverConn, clientConn := net.Pipe()
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{fakeCert(t)},
+	}
+	if isTLS {
+		serverConn = tls.Server(serverConn, tlsConfig)
+		clientConn = tls.Client(clientConn, &tls.Config{InsecureSkipVerify: true})
+	}
+
+	done := make(chan struct{})
+	connCounter++
+	cid := connCounter
+	go func() {
+		serve("test", cid, tlsConfig, serverConn, isTLS, allowLoginWithoutTLS)
+		close(switchDone)
+		close(done)
+	}()
+	client, err := imapclient.New(clientConn, true)
+	tcheck(t, err, "new client")
+	return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn}
+}
+
+func fakeCert(t *testing.T) tls.Certificate {
+	privKey := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) // Fake key, don't use this for real!
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(1), // Required field...
+	}
+	localCertBuf, err := x509.CreateCertificate(cryptorand.Reader, template, template, privKey.Public(), privKey)
+	if err != nil {
+		t.Fatalf("making certificate: %s", err)
+	}
+	cert, err := x509.ParseCertificate(localCertBuf)
+	if err != nil {
+		t.Fatalf("parsing generated certificate: %s", err)
+	}
+	c := tls.Certificate{
+		Certificate: [][]byte{localCertBuf},
+		PrivateKey:  privKey,
+		Leaf:        cert,
+	}
+	return c
+}
+
+func TestLogin(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.transactf("bad", "login too many args")
+	tc.transactf("bad", "login") // no args
+	tc.transactf("no", "login mjl@mox.example badpass")
+	tc.transactf("no", "login mjl testtest") // must use email, not account
+	tc.transactf("no", "login mjl@mox.example test")
+	tc.transactf("no", "login mjl@mox.example testtesttest")
+	tc.transactf("no", `login "mjl@mox.example" "testtesttest"`)
+	tc.transactf("no", "login \"m\xf8x@mox.example\" \"testtesttest\"")
+	tc.transactf("ok", "login mjl@mox.example testtest")
+	tc.close()
+
+	tc = start(t)
+	defer tc.close()
+	tc.transactf("ok", `login "mjl@mox.example" "testtest"`)
+
+	tc.transactf("bad", "logout badarg")
+	tc.transactf("ok", "logout")
+}
+
+// Test that commands don't work in the states they are not supposed to.
+func TestState(t *testing.T) {
+	tc := start(t)
+
+	tc.transactf("bad", "boguscommand")
+
+	notAuthenticated := []string{"starttls", "authenticate", "login"}
+	authenticatedOrSelected := []string{"enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub"}
+	selected := []string{"close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge"}
+
+	// Always allowed.
+	tc.transactf("ok", "capability")
+	tc.transactf("ok", "noop")
+	tc.transactf("ok", "logout")
+	tc.close()
+	tc = start(t)
+	defer tc.close()
+
+	// Not authenticated, lots of commands not allowed.
+	for _, cmd := range append(append([]string{}, authenticatedOrSelected...), selected...) {
+		tc.transactf("no", "%s", cmd)
+	}
+
+	// Some commands not allowed when authenticated.
+	tc.transactf("ok", "login mjl@mox.example testtest")
+	for _, cmd := range append(append([]string{}, notAuthenticated...), selected...) {
+		tc.transactf("no", "%s", cmd)
+	}
+}
+
+func TestLiterals(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Create("tmpbox")
+
+	tc.transactf("ok", "rename {6+}\r\ntmpbox {7+}\r\nntmpbox")
+
+	from := "ntmpbox"
+	to := "tmpbox"
+	fmt.Fprint(tc.client, "xtag rename ")
+	tc.client.WriteSyncLiteral(from)
+	fmt.Fprint(tc.client, " ")
+	tc.client.WriteSyncLiteral(to)
+	fmt.Fprint(tc.client, "\r\n")
+	tc.client.LastTag = "xtag"
+	tc.last(tc.client.Response())
+	if tc.lastResult.Status != "OK" {
+		tc.t.Fatalf(`got %q, expected "OK"`, tc.lastResult.Status)
+	}
+}
+
+// Test longer scenario with login, lists, subscribes, status, selects, etc.
+func TestScenario(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.transactf("ok", "login mjl@mox.example testtest")
+
+	tc.transactf("bad", " missingcommand")
+
+	tc.transactf("ok", "examine inbox")
+	tc.transactf("ok", "unselect")
+
+	tc.transactf("ok", "examine inbox")
+	tc.transactf("ok", "close")
+
+	tc.transactf("ok", "select inbox")
+	tc.transactf("ok", "close")
+
+	tc.transactf("ok", "select inbox")
+	tc.transactf("ok", "expunge")
+	tc.transactf("ok", "check")
+
+	tc.transactf("ok", "subscribe inbox")
+	tc.transactf("ok", "unsubscribe inbox")
+	tc.transactf("ok", "subscribe inbox")
+
+	tc.transactf("ok", `lsub "" "*"`)
+
+	tc.transactf("ok", `list "" ""`)
+	tc.transactf("ok", `namespace`)
+
+	tc.transactf("ok", "enable utf8=accept")
+	tc.transactf("ok", "enable imap4rev2 utf8=accept")
+
+	tc.transactf("no", "create inbox")
+	tc.transactf("ok", "create tmpbox")
+	tc.transactf("ok", "rename tmpbox ntmpbox")
+	tc.transactf("ok", "delete ntmpbox")
+
+	tc.transactf("ok", "status inbox (uidnext messages uidvalidity deleted size unseen recent)")
+
+	tc.transactf("ok", "append inbox (\\seen) {%d+}\r\n%s", len(exampleMsg), exampleMsg)
+	tc.transactf("no", "append bogus () {%d}", len(exampleMsg))
+	tc.cmdf("", "append inbox () {%d}", len(exampleMsg))
+	tc.readprefixline("+")
+	_, err := tc.conn.Write([]byte(exampleMsg + "\r\n"))
+	tc.check(err, "write message")
+	tc.response("ok")
+
+	tc.transactf("ok", "fetch 1 all")
+	tc.transactf("ok", "fetch 1 body")
+	tc.transactf("ok", "fetch 1 binary[]")
+
+	tc.transactf("ok", `store 1 flags (\seen \answered)`)
+	tc.transactf("ok", `store 1 +flags ($junk)`) // should train as junk.
+	tc.transactf("ok", `store 1 -flags ($junk)`) // should retrain as non-junk.
+	tc.transactf("ok", `store 1 -flags (\seen)`) // should untrain completely.
+	tc.transactf("ok", `store 1 -flags (\answered)`)
+	tc.transactf("ok", `store 1 +flags (\answered)`)
+	tc.transactf("ok", `store 1 flags.silent (\seen \answered)`)
+	tc.transactf("ok", `store 1 -flags.silent (\answered)`)
+	tc.transactf("ok", `store 1 +flags.silent (\answered)`)
+	tc.transactf("no", `store 1 flags (\badflag)`)
+	tc.transactf("ok", "noop")
+
+	tc.transactf("ok", "copy 1 Trash")
+	tc.transactf("ok", "copy 1 Trash")
+	tc.transactf("ok", "move 1 Trash")
+
+	tc.transactf("ok", "close")
+	tc.transactf("ok", "select Trash")
+	tc.transactf("ok", `store 1 flags (\deleted)`)
+	tc.transactf("ok", "expunge")
+	tc.transactf("ok", "noop")
+
+	tc.transactf("ok", `store 1 flags (\deleted)`)
+	tc.transactf("ok", "close")
+	tc.transactf("ok", "delete Trash")
+}
+
+func TestMailbox(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	invalid := []string{
+		"e\u0301", // é but as e + acute, not unicode-normalized
+		"/leadingslash",
+		"a//b",
+		"Inbox/",
+		"\x01",
+		" ",
+		"\x7f",
+		"\x80",
+		"\u2028",
+		"\u2029",
+	}
+	for _, bad := range invalid {
+		tc.transactf("no", "select {%d+}\r\n%s", len(bad), bad)
+	}
+}
+
+func TestMailboxDeleted(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", "testtest")
+
+	tc.client.Create("testbox")
+	tc2.client.Select("testbox")
+	tc.client.Delete("testbox")
+
+	// Now try to operate on testbox while it has been removed.
+	tc2.transactf("no", "check")
+	tc2.transactf("no", "expunge")
+	tc2.transactf("no", "uid expunge 1")
+	tc2.transactf("no", "search all")
+	tc2.transactf("no", "uid search all")
+	tc2.transactf("no", "fetch 1:* all")
+	tc2.transactf("no", "uid fetch 1 all")
+	tc2.transactf("no", "store 1 flags ()")
+	tc2.transactf("no", "uid store 1 flags ()")
+	tc2.transactf("bad", "copy 1 inbox") // msgseq 1 not available.
+	tc2.transactf("no", "uid copy 1 inbox")
+	tc2.transactf("bad", "move 1 inbox") // msgseq 1 not available.
+	tc2.transactf("no", "uid move 1 inbox")
+
+	tc2.transactf("ok", "unselect")
+
+	tc.client.Create("testbox")
+	tc2.client.Select("testbox")
+	tc.client.Delete("testbox")
+	tc2.transactf("ok", "close")
+}
+
+func TestID(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	tc.transactf("ok", "id nil")
+	tc.xuntagged(imapclient.UntaggedID{"name": "mox", "version": moxvar.Version})
+
+	tc.transactf("ok", `id ("name" "mox" "version" "1.2.3" "other" "test" "test" nil)`)
+	tc.xuntagged(imapclient.UntaggedID{"name": "mox", "version": moxvar.Version})
+
+	tc.transactf("bad", `id ("name" "mox" "name" "mox")`) // Duplicate field.
+}
+
+func TestSequence(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+
+	tc.transactf("bad", "fetch * all") // ../rfc/9051:7018
+	tc.transactf("bad", "fetch 1 all") // ../rfc/9051:7018
+
+	tc.transactf("ok", "uid fetch 1 all") // non-existing messages are OK for uids.
+	tc.transactf("ok", "uid fetch * all") // * is like uidnext, a non-existing message.
+
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+	tc.transactf("ok", "fetch 2:1,1 uid") // We reorder 2:1 to 1:2, but we don't deduplicate numbers.
+	tc.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1)}},
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2)}},
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1)}},
+	)
+
+	tc.transactf("ok", "uid fetch 3:* uid") // Because * is the last message, which is 2, the range becomes 3:2, which matches the last message.
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2)}})
+}
+
+// Test that a message that is expunged by another session can be read as long as a
+// reference is held by a session. New sessions do not see the expunged message.
+// todo: possibly implement the additional reference counting. so far it hasn't been worth the trouble.
+func disabledTestReference(t *testing.T) {
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Select("inbox")
+	tc.client.Append("inbox", nil, nil, []byte(exampleMsg))
+
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("inbox")
+
+	tc.client.StoreFlagsSet("1", true, `\Deleted`)
+	tc.client.Expunge()
+
+	tc3 := startNoSwitchboard(t)
+	defer tc3.close()
+	tc3.client.Login("mjl@mox.example", "testtest")
+	tc3.transactf("ok", `list "" "inbox" return (status (messages))`)
+	tc3.xuntagged(imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 0}})
+
+	tc2.transactf("ok", "fetch 1 rfc822.size")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchRFC822Size(len(exampleMsg))}})
+}

imapserver/starttls_test.go

@@ -0,0 +1,28 @@
+package imapserver
+
+import (
+	"crypto/tls"
+	"encoding/base64"
+	"testing"
+)
+
+func TestStarttls(t *testing.T) {
+	tc := start(t)
+	tc.client.Starttls(&tls.Config{InsecureSkipVerify: true})
+	tc.transactf("bad", "starttls") // TLS already active.
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.close()
+
+	tc = startArgs(t, true, true, false)
+	tc.transactf("bad", "starttls") // TLS already active.
+	tc.close()
+
+	tc = startArgs(t, true, false, false)
+	tc.transactf("no", `login "mjl@mox.example" "testtest"`)
+	tc.xcode("PRIVACYREQUIRED")
+	tc.transactf("no", "authenticate PLAIN %s", base64.StdEncoding.EncodeToString([]byte("\u0000mjl@mox.example\u0000testtest")))
+	tc.xcode("PRIVACYREQUIRED")
+	tc.client.Starttls(&tls.Config{InsecureSkipVerify: true})
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.close()
+}

imapserver/status_test.go

@@ -0,0 +1,34 @@
+package imapserver
+
+import (
+	"testing"
+
+	"github.com/mjl-/mox/imapclient"
+)
+
+func TestStatus(t *testing.T) {
+	defer mockUIDValidity()()
+	tc := start(t)
+	defer tc.close()
+
+	tc.client.Login("mjl@mox.example", "testtest")
+
+	tc.transactf("bad", "status")                      // Missing param.
+	tc.transactf("bad", "status inbox")                // Missing param.
+	tc.transactf("bad", "status inbox ()")             // At least one attribute required.
+	tc.transactf("bad", "status inbox (uidvalidity) ") // Leftover data.
+	tc.transactf("bad", "status inbox (unknown)")      // Unknown attribute.
+
+	tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 0, "UIDVALIDITY": 1, "UIDNEXT": 1, "UNSEEN": 0, "DELETED": 0, "SIZE": 0, "RECENT": 0, "APPENDLIMIT": 0}})
+
+	// Again, now with a message in the mailbox.
+	tc.transactf("ok", "append inbox {4+}\r\ntest")
+	tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 1, "UIDVALIDITY": 1, "UIDNEXT": 2, "UNSEEN": 1, "DELETED": 0, "SIZE": 4, "RECENT": 0, "APPENDLIMIT": 0}})
+
+	tc.client.Select("inbox")
+	tc.client.StoreFlagsSet("1", true, `\Deleted`)
+	tc.transactf("ok", "status inbox (messages uidnext uidvalidity unseen deleted size recent appendlimit)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 1, "UIDVALIDITY": 1, "UIDNEXT": 2, "UNSEEN": 1, "DELETED": 1, "SIZE": 4, "RECENT": 0, "APPENDLIMIT": 0}})
+}