add basic webserver that can do most of what i need

- serve static files, serving index.html or optionally listings for directories
- redirects
- reverse-proxy, forwarding requests to a backend

these are configurable through the config file. a domain and path regexp have to
be configured. path prefixes can be stripped.  configured domains are added to
the autotls allowlist, so acme automatically fetches certificates for them.

all webserver requests now have (access) logging, metrics, rate limiting.
on http errors, the error message prints an encrypted cid for relating with log files.

this also adds a new mechanism for example config files.

autotls/autotls.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 
@@ -180,12 +181,22 @@ func Load(name, acmeDir, contactEmail, directoryURL string, shutdown <-chan stru
 	return a, nil
 }
 
-// AllowHostname adds hostname for use with ACME.
+// SetAllowedHostnames sets a new list of allowed hostnames for automatic TLS.
-func (m *Manager) AllowHostname(hostname dns.Domain) {
+func (m *Manager) SetAllowedHostnames(hostnames map[dns.Domain]struct{}) {
 	m.Lock()
 	defer m.Unlock()
-	xlog.Debug("autotls add hostname", mlog.Field("hostname", hostname))
+
-	m.hosts[hostname] = struct{}{}
+	// Log as slice, sorted.
+	l := make([]dns.Domain, 0, len(hostnames))
+	for d := range hostnames {
+		l = append(l, d)
+	}
+	sort.Slice(l, func(i, j int) bool {
+		return l[i].Name() < l[j].Name()
+	})
+
+	xlog.Debug("autotls setting allowed hostnames", mlog.Field("hostnames", l))
+	m.hosts = hostnames
 }
 
 // Hostnames returns the allowed host names for use with ACME.

autotls/autotls_test.go

@@ -28,7 +28,7 @@ func TestAutotls(t *testing.T) {
 	if err := m.HostPolicy(context.Background(), "mox.example"); err == nil || !errors.Is(err, errHostNotAllowed) {
 		t.Fatalf("hostpolicy, got err %v, expected errHostNotAllowed", err)
 	}
-	m.AllowHostname(dns.Domain{ASCII: "mox.example"})
+	m.SetAllowedHostnames(map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}})
 	l = m.Hostnames()
 	if !reflect.DeepEqual(l, []dns.Domain{{ASCII: "mox.example"}}) {
 		t.Fatalf("hostnames, got %v, expected single mox.example", l)
@@ -79,7 +79,7 @@ func TestAutotls(t *testing.T) {
 		t.Fatalf("private key changed after reload")
 	}
 	m.shutdown = make(chan struct{})
-	m.AllowHostname(dns.Domain{ASCII: "mox.example"})
+	m.SetAllowedHostnames(map[dns.Domain]struct{}{{ASCII: "mox.example"}: {}})
 	if err := m.HostPolicy(context.Background(), "mox.example"); err != nil {
 		t.Fatalf("hostpolicy, got err %v, expected no error", err)
 	}

config/config.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"net"
+	"net/url"
 	"reflect"
 	"regexp"
 	"time"
@@ -16,7 +17,7 @@ import (
 	"github.com/mjl-/mox/smtp"
 )
 
-// todo: better default values, so less has to be specified in the config file. junkfilter and rejects mailbox should be enabled by default. other features as well possibly.
+// todo: better default values, so less has to be specified in the config file.
 
 // Port returns port if non-zero, and fallback otherwise.
 func Port(port, fallback int) int {
@@ -69,8 +70,9 @@ type Static struct {
 
 // Dynamic is the parsed form of domains.conf, and is automatically reloaded when changed.
 type Dynamic struct {
-	Domains  map[string]Domain  `sconf-doc:"Domains for which email is accepted. For internationalized domains, use their IDNA names in UTF-8."`
+	Domains     map[string]Domain  `sconf-doc:"Domains for which email is accepted. For internationalized domains, use their IDNA names in UTF-8."`
-	Accounts map[string]Account `sconf-doc:"Accounts to which email can be delivered. An account can accept email for multiple domains, for multiple localparts, and deliver to multiple mailboxes."`
+	Accounts    map[string]Account `sconf-doc:"Accounts to which email can be delivered. An account can accept email for multiple domains, for multiple localparts, and deliver to multiple mailboxes."`
+	WebHandlers []WebHandler       `sconf:"optional" sconf-doc:"Handle webserver requests by serving static files, redirecting or reverse-proxying HTTP(s). The first matching WebHandler will handle the request. Built-in handlers for autoconfig and mta-sts always run first. If no handler matches, the response status code is file not found (404). If functionality you need is missng, simply forward the requests to an application that can provide the needed functionality."`
 }
 
 type ACME struct {
@@ -149,6 +151,14 @@ type Listener struct {
 		Port    int  `sconf:"optional" sconf-doc:"TLS port, 443 by default. You should only override this if you cannot listen on port 443 directly. MTA-STS requests will be made to port 443, so you'll have to add an external mechanism to get the connection here, e.g. by configuring port forwarding."`
 		NonTLS  bool `sconf:"optional" sconf-doc:"If set, plain HTTP instead of HTTPS is spoken on the configured port. Can be useful when the mta-sts domain is reverse proxied."`
 	} `sconf:"optional" sconf-doc:"Serve MTA-STS policies describing SMTP TLS requirements. Requires a TLS config."`
+	WebserverHTTP struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Port for plain HTTP (non-TLS) webserver."`
+	} `sconf:"optional" sconf-doc:"All configured WebHandlers will serve on an enabled listener."`
+	WebserverHTTPS struct {
+		Enabled bool
+		Port    int `sconf:"optional" sconf-doc:"Port for HTTPS webserver."`
+	} `sconf:"optional" sconf-doc:"All configured WebHandlers will serve on an enabled listener. Either ACME must be configured, or for each WebHandler domain a TLS certificate must be configured."`
 }
 
 type Domain struct {
@@ -296,3 +306,44 @@ type TLS struct {
 	Config     *tls.Config `sconf:"-" json:"-"` // TLS config for non-ACME-verification connections, i.e. SMTP and IMAP, and not port 443.
 	ACMEConfig *tls.Config `sconf:"-" json:"-"` // TLS config that handles ACME verification, for serving on port 443.
 }
+
+type WebHandler struct {
+	LogName    string `sconf:"optional" sconf-doc:"Name to use in logging and metrics."`
+	Domain     string `sconf-doc:"Both Domain and PathRegexp must match for this WebHandler to match a request. Exactly one of WebStatic, WebRedirect, WebForward must be set."`
+	PathRegexp string `sconf-doc:"Regular expression matched against request path, must always start with ^ to ensure matching from the start of the path. The matching prefix can optionally be stripped by WebForward. The regular expression does not have to end with $."`
+
+	DontRedirectPlainHTTP bool         `sconf:"optional" sconf-doc:"If set, plain HTTP requests are not automatically permanently redirected (308) to HTTPS. If you don't have a HTTPS webserver configured, set this to true."`
+	WebStatic             *WebStatic   `sconf:"optional" sconf-doc:"Serve static files."`
+	WebRedirect           *WebRedirect `sconf:"optional" sconf-doc:"Redirect requests to configured URL."`
+	WebForward            *WebForward  `sconf:"optional" sconf-doc:"Forward requests to another webserver, i.e. reverse proxy."`
+
+	Name      string         `sconf:"-"` // Either LogName, or numeric index if LogName was empty. Used instead of LogName in logging/metrics.
+	DNSDomain dns.Domain     `sconf:"-"`
+	Path      *regexp.Regexp `sconf:"-" json:"-"`
+}
+
+type WebStatic struct {
+	StripPrefix      string            `sconf:"optional" sconf-doc:"Path to strip from the request URL before evaluating to a local path. If the requested URL path does not start with this prefix and ContinueNotFound it is considered non-matching and next WebHandlers are tried. If ContinueNotFound is not set, a file not found (404) is returned in that case."`
+	Root             string            `sconf-doc:"Directory to serve files from for this handler. Keep in mind that relative paths are relative to the working directory of mox."`
+	ListFiles        bool              `sconf:"optional" sconf-doc:"If set, and a directory is requested, and no index.html is present that can be served, a file listing is returned. Results in 403 if ListFiles is not set. If a directory is requested and the URL does not end with a slash, the response is a redirect to the path with trailing slash."`
+	ContinueNotFound bool              `sconf:"optional" sconf-doc:"If a requested URL does not exist, don't return a file not found (404) response, but consider this handler non-matching and continue attempts to serve with later WebHandlers, which may be a reverse proxy generating dynamic content, possibly even writing a static file for a next request to serve statically. If ContinueNotFound is set, HTTP requests other than GET and HEAD do not match. This mechanism can be used to implement the equivalent of 'try_files' in other webservers."`
+	ResponseHeaders  map[string]string `sconf:"optional" sconf-doc:"Headers to add to the response. Useful for cache-control, content-type, etc. By default, Content-Type headers are automatically added for recognized file types, unless added explicitly through this setting. For directory listings, a content-type header is skipped."`
+}
+
+type WebRedirect struct {
+	BaseURL        string `sconf:"optional" sconf-doc:"Base URL to redirect to. The path must be empty and will be replaced, either by the request URL path, or byOrigPathRegexp/ReplacePath. Scheme, host, port and fragment stay intact, and query strings are combined. If empty, the response redirects to a different path through OrigPathRegexp and ReplacePath, which must then be set. Use a URL without scheme to redirect without changing the protocol, e.g. //newdomain/."`
+	OrigPathRegexp string `sconf:"optional" sconf-doc:"Regular expression for matching path. If set and path does not match, a 404 is returned. The HTTP path used for matching always starts with a slash."`
+	ReplacePath    string `sconf:"optional" sconf-doc:"Replacement path for destination URL based on OrigPathRegexp. Implemented with Go's Regexp.ReplaceAllString: $1 is replaced with the text of the first submatch, etc. If both OrigPathRegexp and ReplacePath are empty, BaseURL must be set and all paths are redirected unaltered."`
+	StatusCode     int    `sconf:"optional" sconf-doc:"Status code to use in redirect, e.g. 307. By default, a permanent redirect (308) is returned."`
+
+	URL      *url.URL       `sconf:"-" json:"-"`
+	OrigPath *regexp.Regexp `sconf:"-" json:"-"`
+}
+
+type WebForward struct {
+	StripPath       bool              `sconf:"optional" sconf-doc:"Strip the matching WebHandler path from the WebHandler before forwarding the request."`
+	URL             string            `sconf-doc:"URL to forward HTTP requests to, e.g. http://127.0.0.1:8123/base. If StripPath is false the full request path is added to the URL. Host headers are sent unmodified. New X-Forwarded-{For,Host,Proto} headers are set. Any query string in the URL is ignored. Requests are made using Go's net/http.DefaultTransport that takes environment variables HTTP_PROXY and HTTPS_PROXY into account."`
+	ResponseHeaders map[string]string `sconf:"optional" sconf-doc:"Headers to add to the response. Useful for adding security- and cache-related headers."`
+
+	TargetURL *url.URL `sconf:"-" json:"-"`
+}

config/doc.go

@@ -269,6 +269,22 @@ describe-static" and "mox config describe-domains":
 				# useful when the mta-sts domain is reverse proxied. (optional)
 				NonTLS: false
 
+			# All configured WebHandlers will serve on an enabled listener. (optional)
+			WebserverHTTP:
+				Enabled: false
+
+				# Port for plain HTTP (non-TLS) webserver. (optional)
+				Port: 0
+
+			# All configured WebHandlers will serve on an enabled listener. Either ACME must
+			# be configured, or for each WebHandler domain a TLS certificate must be
+			# configured. (optional)
+			WebserverHTTPS:
+				Enabled: false
+
+				# Port for HTTPS webserver. (optional)
+				Port: 0
+
 	# Destination for emails delivered to postmaster address.
 	Postmaster:
 		Account:
@@ -540,6 +556,176 @@ describe-static" and "mox config describe-domains":
 					# Occurrences in word database until a word is considered rare and its influence
 					# in calculating probability reduced. E.g. 1 or 2. (optional)
 					RareWords: 0
+
+	# Handle webserver requests by serving static files, redirecting or
+	# reverse-proxying HTTP(s). The first matching WebHandler will handle the request.
+	# Built-in handlers for autoconfig and mta-sts always run first. If no handler
+	# matches, the response status code is file not found (404). If functionality you
+	# need is missng, simply forward the requests to an application that can provide
+	# the needed functionality. (optional)
+	WebHandlers:
+		-
+
+			# Name to use in logging and metrics. (optional)
+			LogName:
+
+			# Both Domain and PathRegexp must match for this WebHandler to match a request.
+			# Exactly one of WebStatic, WebRedirect, WebForward must be set.
+			Domain:
+
+			# Regular expression matched against request path, must always start with ^ to
+			# ensure matching from the start of the path. The matching prefix can optionally
+			# be stripped by WebForward. The regular expression does not have to end with $.
+			PathRegexp:
+
+			# If set, plain HTTP requests are not automatically permanently redirected (308)
+			# to HTTPS. If you don't have a HTTPS webserver configured, set this to true.
+			# (optional)
+			DontRedirectPlainHTTP: false
+
+			# Serve static files. (optional)
+			WebStatic:
+
+				# Path to strip from the request URL before evaluating to a local path. If the
+				# requested URL path does not start with this prefix and ContinueNotFound it is
+				# considered non-matching and next WebHandlers are tried. If ContinueNotFound is
+				# not set, a file not found (404) is returned in that case. (optional)
+				StripPrefix:
+
+				# Directory to serve files from for this handler. Keep in mind that relative paths
+				# are relative to the working directory of mox.
+				Root:
+
+				# If set, and a directory is requested, and no index.html is present that can be
+				# served, a file listing is returned. Results in 403 if ListFiles is not set. If a
+				# directory is requested and the URL does not end with a slash, the response is a
+				# redirect to the path with trailing slash. (optional)
+				ListFiles: false
+
+				# If a requested URL does not exist, don't return a file not found (404) response,
+				# but consider this handler non-matching and continue attempts to serve with later
+				# WebHandlers, which may be a reverse proxy generating dynamic content, possibly
+				# even writing a static file for a next request to serve statically. If
+				# ContinueNotFound is set, HTTP requests other than GET and HEAD do not match.
+				# This mechanism can be used to implement the equivalent of 'try_files' in other
+				# webservers. (optional)
+				ContinueNotFound: false
+
+				# Headers to add to the response. Useful for cache-control, content-type, etc. By
+				# default, Content-Type headers are automatically added for recognized file types,
+				# unless added explicitly through this setting. For directory listings, a
+				# content-type header is skipped. (optional)
+				ResponseHeaders:
+					x:
+
+			# Redirect requests to configured URL. (optional)
+			WebRedirect:
+
+				# Base URL to redirect to. The path must be empty and will be replaced, either by
+				# the request URL path, or byOrigPathRegexp/ReplacePath. Scheme, host, port and
+				# fragment stay intact, and query strings are combined. If empty, the response
+				# redirects to a different path through OrigPathRegexp and ReplacePath, which must
+				# then be set. Use a URL without scheme to redirect without changing the protocol,
+				# e.g. //newdomain/. (optional)
+				BaseURL:
+
+				# Regular expression for matching path. If set and path does not match, a 404 is
+				# returned. The HTTP path used for matching always starts with a slash. (optional)
+				OrigPathRegexp:
+
+				# Replacement path for destination URL based on OrigPathRegexp. Implemented with
+				# Go's Regexp.ReplaceAllString: $1 is replaced with the text of the first
+				# submatch, etc. If both OrigPathRegexp and ReplacePath are empty, BaseURL must be
+				# set and all paths are redirected unaltered. (optional)
+				ReplacePath:
+
+				# Status code to use in redirect, e.g. 307. By default, a permanent redirect (308)
+				# is returned. (optional)
+				StatusCode: 0
+
+			# Forward requests to another webserver, i.e. reverse proxy. (optional)
+			WebForward:
+
+				# Strip the matching WebHandler path from the WebHandler before forwarding the
+				# request. (optional)
+				StripPath: false
+
+				# URL to forward HTTP requests to, e.g. http://127.0.0.1:8123/base. If StripPath
+				# is false the full request path is added to the URL. Host headers are sent
+				# unmodified. New X-Forwarded-{For,Host,Proto} headers are set. Any query string
+				# in the URL is ignored. Requests are made using Go's net/http.DefaultTransport
+				# that takes environment variables HTTP_PROXY and HTTPS_PROXY into account.
+				URL:
+
+				# Headers to add to the response. Useful for adding security- and cache-related
+				# headers. (optional)
+				ResponseHeaders:
+					x:
+
+# Examples
+
+Mox includes configuration files to illustrate common setups. You can see these
+examples with "mox examples", and print a specific example with "mox examples
+<name>". Below are all examples included in mox.
+
+# Example webhandlers
+
+	# Snippet of domains.conf to configure WebHandlers.
+	WebHandlers:
+		-
+			# The name of the handler, used in logging and metrics.
+			LogName: staticmjl
+			# With ACME configured, each configured domain will automatically get a TLS
+			# certificate on first request.
+			Domain: mox.example
+			PathRegexp: ^/who/mjl/
+			WebStatic:
+				StripPrefix: /who/mjl
+				# Requested path /who/mjl/inferno/ resolves to local web/mjl/inferno.
+				# If a directory contains an index.html, it is served when a directory is requested.
+				Root: web/mjl
+				# With ListFiles true, if a directory does not contain an index.html, the contents are listed.
+				ListFiles: true
+				ResponseHeaders:
+					X-Mox: hi
+		-
+			LogName: redir
+			Domain: mox.example
+			PathRegexp: ^/redir/a/b/c
+			# Don't redirect from plain HTTP to HTTPS.
+			DontRedirectPlainHTTP: true
+			WebRedirect:
+				# Just change the domain and add query string set fragment. No change to scheme.
+				# Path will start with /redir/a/b/c (and whathever came after) because no
+				# OrigPathRegexp+ReplacePath is set.
+				BaseURL: //moxest.example?q=1#frag
+				# Default redirection is 308 - Permanent Redirect.
+				StatusCode: 307
+		-
+			LogName: oldnew
+			Domain: mox.example
+			PathRegexp: ^/old/
+			WebRedirect:
+				# Replace path, leaving rest of URL intact.
+				OrigPathRegexp: ^/old/(.*)
+				ReplacePath: /new/$1
+		-
+			LogName: app
+			Domain: mox.example
+			PathRegexp: ^/app/
+			WebForward:
+				# Strip the path matched by PathRegexp before forwarding the request. So original
+				# request /app/api become just /api.
+				StripPath: true
+				# URL of backend, where requests are forwarded to. The path in the URL is kept,
+				# so for incoming request URL /app/api, the outgoing request URL has path /app-v2/api.
+				# Requests are made with Go's net/http DefaultTransporter, including using
+				# HTTP_PROXY and HTTPS_PROXY environment variables.
+				URL: http://127.0.0.1:8900/app-v2/
+				# Add headers to response.
+				ResponseHeaders:
+					X-Frame-Options: deny
+					X-Content-Type-Options: nosniff
 */
 package config
 

doc.go

@@ -41,6 +41,7 @@ low-maintenance self-hosted email.
 	mox config domain rm domain
 	mox config describe-sendmail >/etc/moxsubmit.conf
 	mox config printservice >mox.service
+	mox examples [name]
 	mox checkupdate
 	mox cid cid
 	mox clientconfig domain
@@ -388,6 +389,12 @@ date version.
 
 	usage: mox config printservice >mox.service
 
+# mox examples
+
+List available examples, or print a specific example.
+
+	usage: mox examples [name]
+
 # mox checkupdate
 
 Check if a newer version of mox is available.

gendoc.sh

@@ -61,6 +61,23 @@ cat <<EOF
 EOF
 ./mox config describe-domains | sed 's/^/\t/'
 
+cat <<EOF
+
+# Examples
+
+Mox includes configuration files to illustrate common setups. You can see these
+examples with "mox examples", and print a specific example with "mox examples
+<name>". Below are all examples included in mox.
+
+EOF
+
+for ex in $(./mox examples); do
+	echo '# Example '$ex
+	echo
+	./mox examples $ex | sed 's/^/\t/'
+	echo
+done
+
 cat <<EOF
 */
 package config

http/mtasts.go

@@ -13,7 +13,9 @@ import (
 )
 
 func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
-	log := xlog.WithCid(mox.Cid())
+	log := func() *mlog.Log {
+		return xlog.WithContext(r.Context())
+	}
 
 	host := strings.ToLower(r.Host)
 	if !strings.HasPrefix(host, "mta-sts.") {
@@ -28,7 +30,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
 	}
 	domain, err := dns.ParseDomain(host)
 	if err != nil {
-		log.Errorx("mtasts policy request: bad domain", err, mlog.Field("host", host))
+		log().Errorx("mtasts policy request: bad domain", err, mlog.Field("host", host))
 		http.NotFound(w, r)
 		return
 	}
@@ -49,7 +51,7 @@ func mtastsPolicyHandle(w http.ResponseWriter, r *http.Request) {
 		}
 		d, err := dns.ParseDomain(s)
 		if err != nil {
-			log.Errorx("bad domain in mtasts config", err, mlog.Field("domain", s))
+			log().Errorx("bad domain in mtasts config", err, mlog.Field("domain", s))
 			http.Error(w, "500 - internal server error - invalid domain in configuration", http.StatusInternalServerError)
 			return
 		}

http/web.go

@@ -4,27 +4,119 @@
 package http
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	golog "log"
 	"net"
 	"net/http"
 	"os"
+	"path"
+	"sort"
 	"strings"
 	"time"
 
 	_ "net/http/pprof"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/mjl-/mox/config"
 	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/ratelimit"
 )
 
 var xlog = mlog.New("http")
 
+var metricHTTPServer = promauto.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name:    "mox_httpserver_request_duration_seconds",
+		Help:    "HTTP(s) server request with handler name, protocol, method, result codes, and duration in seconds.",
+		Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30, 60, 120},
+	},
+	[]string{
+		"handler", // Name from webhandler, can be empty.
+		"proto",   // "http" or "https"
+		"method",  // "(unknown)" and otherwise only common verbs
+		"code",
+	},
+)
+
+// http.ResponseWriter that writes access log and tracks metrics at end of response.
+type loggingWriter struct {
+	W     http.ResponseWriter // Calls are forwarded.
+	Start time.Time
+	R     *http.Request
+
+	Handler string // Set by router.
+
+	// Set by handlers.
+	Code     int
+	Size     int64
+	WriteErr error
+}
+
+func (w *loggingWriter) Header() http.Header {
+	return w.W.Header()
+}
+
+func (w *loggingWriter) Write(buf []byte) (int, error) {
+	n, err := w.W.Write(buf)
+	if n > 0 {
+		w.Size += int64(n)
+	}
+	if err != nil && w.WriteErr == nil {
+		w.WriteErr = err
+	}
+	return n, err
+}
+
+func (w *loggingWriter) WriteHeader(statusCode int) {
+	if w.Code == 0 {
+		w.Code = statusCode
+	}
+	w.W.WriteHeader(statusCode)
+}
+
+var tlsVersions = map[uint16]string{
+	tls.VersionTLS10: "tls1.0",
+	tls.VersionTLS11: "tls1.1",
+	tls.VersionTLS12: "tls1.2",
+	tls.VersionTLS13: "tls1.3",
+}
+
+func metricHTTPMethod(method string) string {
+	// https://www.iana.org/assignments/http-methods/http-methods.xhtml
+	method = strings.ToLower(method)
+	switch method {
+	case "acl", "baseline-control", "bind", "checkin", "checkout", "connect", "copy", "delete", "get", "head", "label", "link", "lock", "merge", "mkactivity", "mkcalendar", "mkcol", "mkredirectref", "mkworkspace", "move", "options", "orderpatch", "patch", "post", "pri", "propfind", "proppatch", "put", "rebind", "report", "search", "trace", "unbind", "uncheckout", "unlink", "unlock", "update", "updateredirectref", "version-control":
+		return method
+	}
+	return "(other)"
+}
+
+func (w *loggingWriter) Done() {
+	method := metricHTTPMethod(w.R.Method)
+	proto := "http"
+	if w.R.TLS != nil {
+		proto = "https"
+	}
+	metricHTTPServer.WithLabelValues(w.Handler, proto, method, fmt.Sprintf("%d", w.Code)).Observe(float64(time.Since(w.Start)) / float64(time.Second))
+
+	tlsinfo := "plain"
+	if w.R.TLS != nil {
+		if v, ok := tlsVersions[w.R.TLS.Version]; ok {
+			tlsinfo = v
+		} else {
+			tlsinfo = "(other)"
+		}
+	}
+	xlog.WithContext(w.R.Context()).Debugx("http request", w.WriteErr, mlog.Field("httpaccess", ""), mlog.Field("handler", w.Handler), mlog.Field("url", w.R.URL), mlog.Field("host", w.R.Host), mlog.Field("duration", time.Since(w.Start)), mlog.Field("size", w.Size), mlog.Field("statuscode", w.Code), mlog.Field("proto", strings.ToLower(w.R.Proto)), mlog.Field("remoteaddr", w.R.RemoteAddr), mlog.Field("tlsinfo", tlsinfo))
+}
+
 // Set some http headers that should prevent potential abuse. Better safe than sorry.
 func safeHeaders(fn http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -37,68 +129,166 @@ func safeHeaders(fn http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// Built-in handlers, e.g. mta-sts and autoconfig.
+type pathHandler struct {
+	Name string // For logging/metrics.
+	Path string // Path to register, like on http.ServeMux.
+	Fn   http.HandlerFunc
+}
+type serve struct {
+	Kinds        []string // Type of handler and protocol (http/https).
+	TLSConfig    *tls.Config
+	PathHandlers []pathHandler // Sorted, longest first.
+	Webserver    bool          // Whether serving WebHandler. PathHandlers are always evaluated before WebHandlers.
+}
+
+// HandleFunc registers a named handler for a path. If path ends with a slash, it
+// is used as prefix match, otherwise a full path match is required.
+func (s *serve) HandleFunc(name, path string, fn http.HandlerFunc) {
+	s.PathHandlers = append(s.PathHandlers, pathHandler{name, path, fn})
+}
+
+var (
+	limiterConnectionrate = &ratelimit.Limiter{
+		WindowLimits: []ratelimit.WindowLimit{
+			{
+				Window: time.Minute,
+				Limits: [...]int64{1000, 3000, 9000},
+			},
+			{
+				Window: time.Hour,
+				Limits: [...]int64{5000, 15000, 45000},
+			},
+		},
+	}
+)
+
+// ServeHTTP is the starting point for serving HTTP requests. It dispatches to the
+// right pathHandler or WebHandler, and it generates access logs and tracks
+// metrics.
+func (s *serve) ServeHTTP(xw http.ResponseWriter, r *http.Request) {
+	now := time.Now()
+	// Rate limiting as early as possible.
+	ipstr, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		xlog.Debugx("split host:port client remoteaddr", err, mlog.Field("remoteaddr", r.RemoteAddr))
+	} else if ip := net.ParseIP(ipstr); ip == nil {
+		xlog.Debug("parsing ip for client remoteaddr", mlog.Field("remoteaddr", r.RemoteAddr))
+	} else if !limiterConnectionrate.Add(ip, now, 1) {
+		method := metricHTTPMethod(r.Method)
+		proto := "http"
+		if r.TLS != nil {
+			proto = "https"
+		}
+		metricHTTPServer.WithLabelValues("(ratelimited)", proto, method, "429").Observe(0)
+		// No logging, that's just noise.
+
+		http.Error(xw, "http 429 - too many auth attempts", http.StatusTooManyRequests)
+		return
+	}
+
+	ctx := context.WithValue(r.Context(), mlog.CidKey, mox.Cid())
+	r = r.WithContext(ctx)
+
+	nw := &loggingWriter{
+		W:     xw,
+		Start: now,
+		R:     r,
+	}
+	defer nw.Done()
+
+	// Cleanup path, removing ".." and ".". Keep any trailing slash.
+	trailingPath := strings.HasSuffix(r.URL.Path, "/")
+	if r.URL.Path == "" {
+		r.URL.Path = "/"
+	}
+	r.URL.Path = path.Clean(r.URL.Path)
+	if r.URL.Path == "." {
+		r.URL.Path = "/"
+	}
+	if trailingPath && !strings.HasSuffix(r.URL.Path, "/") {
+		r.URL.Path += "/"
+	}
+
+	for _, h := range s.PathHandlers {
+		if r.URL.Path == h.Path || strings.HasSuffix(h.Path, "/") && strings.HasPrefix(r.URL.Path, h.Path) {
+			nw.Handler = h.Name
+			h.Fn(nw, r)
+			return
+		}
+	}
+	if s.Webserver {
+		if WebHandle(nw, r) {
+			return
+		}
+	}
+	nw.Handler = "(nomatch)"
+	http.NotFound(nw, r)
+}
+
 // Listen binds to sockets for HTTP listeners, including those required for ACME to
 // generate TLS certificates. It stores the listeners so Serve can start serving them.
 func Listen() {
-	type serve struct {
-		kinds     []string
-		tlsConfig *tls.Config
-		mux       *http.ServeMux
-	}
-
 	for name, l := range mox.Conf.Static.Listeners {
-		portServe := map[int]serve{}
+		portServe := map[int]*serve{}
 
-		var ensureServe func(https bool, port int, kind string) serve
+		var ensureServe func(https bool, port int, kind string) *serve
-		ensureServe = func(https bool, port int, kind string) serve {
+		ensureServe = func(https bool, port int, kind string) *serve {
-			s, ok := portServe[port]
+			s := portServe[port]
-			if !ok {
+			if s == nil {
-				s = serve{nil, nil, &http.ServeMux{}}
+				s = &serve{nil, nil, nil, false}
+				portServe[port] = s
 			}
-			s.kinds = append(s.kinds, kind)
+			s.Kinds = append(s.Kinds, kind)
 			if https && l.TLS.ACME != "" {
-				s.tlsConfig = l.TLS.ACMEConfig
+				s.TLSConfig = l.TLS.ACMEConfig
 			} else if https {
-				s.tlsConfig = l.TLS.Config
+				s.TLSConfig = l.TLS.Config
 				if l.TLS.ACME != "" {
-					ensureServe(true, config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443), "acme-tls-alpn-01")
+					tlsport := config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443)
+					ensureServe(true, tlsport, "acme-tls-alpn-01")
 				}
 			}
-			portServe[port] = s
 			return s
 		}
 
 		if l.TLS != nil && l.TLS.ACME != "" && (l.SMTP.Enabled && !l.SMTP.NoSTARTTLS || l.Submissions.Enabled || l.IMAPS.Enabled) {
-			ensureServe(true, config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443), "acme-tls-alpn01")
+			port := config.Port(mox.Conf.Static.ACME[l.TLS.ACME].Port, 443)
+			ensureServe(true, port, "acme-tls-alpn01")
 		}
 
 		if l.AccountHTTP.Enabled {
-			srv := ensureServe(false, config.Port(l.AccountHTTP.Port, 80), "account-http")
+			port := config.Port(l.AccountHTTP.Port, 80)
-			srv.mux.HandleFunc("/", safeHeaders(accountHandle))
+			srv := ensureServe(false, port, "account-http")
+			srv.HandleFunc("account", "/", safeHeaders(accountHandle))
 		}
 		if l.AccountHTTPS.Enabled {
-			srv := ensureServe(true, config.Port(l.AccountHTTPS.Port, 443), "account-https")
+			port := config.Port(l.AccountHTTPS.Port, 443)
-			srv.mux.HandleFunc("/", safeHeaders(accountHandle))
+			srv := ensureServe(true, port, "account-https")
+			srv.HandleFunc("account", "/", safeHeaders(accountHandle))
 		}
 
 		if l.AdminHTTP.Enabled {
-			srv := ensureServe(false, config.Port(l.AdminHTTP.Port, 80), "admin-http")
+			port := config.Port(l.AdminHTTP.Port, 80)
+			srv := ensureServe(false, port, "admin-http")
 			if !l.AccountHTTP.Enabled {
-				srv.mux.HandleFunc("/", safeHeaders(adminIndex))
+				srv.HandleFunc("admin", "/", safeHeaders(adminIndex))
 			}
-			srv.mux.HandleFunc("/admin/", safeHeaders(adminHandle))
+			srv.HandleFunc("admin", "/admin/", safeHeaders(adminHandle))
 		}
 		if l.AdminHTTPS.Enabled {
-			srv := ensureServe(true, config.Port(l.AdminHTTPS.Port, 443), "admin-https")
+			port := config.Port(l.AdminHTTPS.Port, 443)
+			srv := ensureServe(true, port, "admin-https")
 			if !l.AccountHTTPS.Enabled {
-				srv.mux.HandleFunc("/", safeHeaders(adminIndex))
+				srv.HandleFunc("admin", "/", safeHeaders(adminIndex))
 			}
-			srv.mux.HandleFunc("/admin/", safeHeaders(adminHandle))
+			srv.HandleFunc("admin", "/admin/", safeHeaders(adminHandle))
 		}
 		if l.MetricsHTTP.Enabled {
-			srv := ensureServe(false, config.Port(l.MetricsHTTP.Port, 8010), "metrics-http")
+			port := config.Port(l.MetricsHTTP.Port, 8010)
-			srv.mux.Handle("/metrics", safeHeaders(promhttp.Handler().ServeHTTP))
+			srv := ensureServe(false, port, "metrics-http")
-			srv.mux.HandleFunc("/", safeHeaders(func(w http.ResponseWriter, r *http.Request) {
+			srv.HandleFunc("metrics", "/metrics", safeHeaders(promhttp.Handler().ServeHTTP))
+			srv.HandleFunc("metrics", "/", safeHeaders(func(w http.ResponseWriter, r *http.Request) {
 				if r.URL.Path != "/" {
 					http.NotFound(w, r)
 					return
@@ -111,13 +301,15 @@ func Listen() {
 			}))
 		}
 		if l.AutoconfigHTTPS.Enabled {
-			srv := ensureServe(!l.AutoconfigHTTPS.NonTLS, config.Port(l.AutoconfigHTTPS.Port, 443), "autoconfig-https")
+			port := config.Port(l.AutoconfigHTTPS.Port, 443)
-			srv.mux.HandleFunc("/mail/config-v1.1.xml", safeHeaders(autoconfHandle(l)))
+			srv := ensureServe(!l.AutoconfigHTTPS.NonTLS, port, "autoconfig-https")
-			srv.mux.HandleFunc("/autodiscover/autodiscover.xml", safeHeaders(autodiscoverHandle(l)))
+			srv.HandleFunc("autoconfig", "/mail/config-v1.1.xml", safeHeaders(autoconfHandle(l)))
+			srv.HandleFunc("autodiscover", "/autodiscover/autodiscover.xml", safeHeaders(autodiscoverHandle(l)))
 		}
 		if l.MTASTSHTTPS.Enabled {
-			srv := ensureServe(!l.AutoconfigHTTPS.NonTLS, config.Port(l.MTASTSHTTPS.Port, 443), "mtasts-https")
+			port := config.Port(l.MTASTSHTTPS.Port, 443)
-			srv.mux.HandleFunc("/.well-known/mta-sts.txt", safeHeaders(mtastsPolicyHandle))
+			srv := ensureServe(!l.AutoconfigHTTPS.NonTLS, port, "mtasts-https")
+			srv.HandleFunc("mtasts", "/.well-known/mta-sts.txt", safeHeaders(mtastsPolicyHandle))
 		}
 		if l.PprofHTTP.Enabled {
 			// Importing net/http/pprof registers handlers on the default serve mux.
@@ -125,7 +317,19 @@ func Listen() {
 			if _, ok := portServe[port]; ok {
 				xlog.Fatal("cannot serve pprof on same endpoint as other http services")
 			}
-			portServe[port] = serve{[]string{"pprof-http"}, nil, http.DefaultServeMux}
+			srv := &serve{[]string{"pprof-http"}, nil, nil, false}
+			portServe[port] = srv
+			srv.HandleFunc("pprof", "/", http.DefaultServeMux.ServeHTTP)
+		}
+		if l.WebserverHTTP.Enabled {
+			port := config.Port(l.WebserverHTTP.Port, 80)
+			srv := ensureServe(false, port, "webserver-http")
+			srv.Webserver = true
+		}
+		if l.WebserverHTTPS.Enabled {
+			port := config.Port(l.WebserverHTTPS.Port, 443)
+			srv := ensureServe(true, port, "webserver-https")
+			srv.Webserver = true
 		}
 
 		// We'll explicitly ensure these TLS certs exist (e.g. are created with ACME)
@@ -137,10 +341,8 @@ func Listen() {
 		if l.TLS != nil && l.TLS.ACME != "" {
 			m := mox.Conf.Static.ACME[l.TLS.ACME].Manager
 
-			m.AllowHostname(mox.Conf.Static.HostnameDomain)
 			ensureHosts[mox.Conf.Static.HostnameDomain] = struct{}{}
 			if l.HostnameDomain.ASCII != "" {
-				m.AllowHostname(l.HostnameDomain)
 				ensureHosts[l.HostnameDomain] = struct{}{}
 			}
 
@@ -180,7 +382,17 @@ func Listen() {
 
 		for port, srv := range portServe {
 			for _, ip := range l.IPs {
-				listen1(ip, port, srv.tlsConfig, name, srv.kinds, srv.mux)
+				sort.Slice(srv.PathHandlers, func(i, j int) bool {
+					a := srv.PathHandlers[i].Path
+					b := srv.PathHandlers[j].Path
+					if len(a) == len(b) {
+						// For consistent order.
+						return a < b
+					}
+					// Longest paths first.
+					return len(a) > len(b)
+				})
+				listen1(ip, port, srv.TLSConfig, name, srv.Kinds, srv)
 			}
 		}
 	}
@@ -203,7 +415,7 @@ func adminIndex(w http.ResponseWriter, r *http.Request) {
 var servers []func()
 
 // listen prepares a listener, and adds it to "servers", to be launched (if not running as root) through Serve.
-func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []string, mux *http.ServeMux) {
+func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []string, handler http.Handler) {
 	addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
 
 	var protocol string
@@ -231,7 +443,7 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st
 	}
 
 	server := &http.Server{
-		Handler:   mux,
+		Handler:   handler,
 		TLSConfig: tlsConfig,
 		ErrorLog:  golog.New(mlog.ErrWriter(xlog.Fields(mlog.Field("pkg", "net/http")), mlog.LevelInfo, protocol+" error"), "", 0),
 	}

http/webserver.go

@@ -0,0 +1,385 @@
+package http
+
+import (
+	"fmt"
+	htmltemplate "html/template"
+	"io"
+	golog "log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxio"
+)
+
+// todo: automatic gzip on responses, if client supports it, and if content looks compressible.
+
+// WebHandle serves an HTTP request by going through the list of WebHandlers,
+// check if there is a domain+path match, and running the handler if so.
+// WebHandle runs after the built-in handlers for mta-sts, autoconfig, etc.
+// If no handler matched, false is returned.
+// WebHandle sets w.Name to that of the matching handler.
+func WebHandle(w *loggingWriter, r *http.Request) (handled bool) {
+	log := func() *mlog.Log {
+		return xlog.WithContext(r.Context())
+	}
+
+	host, _, err := net.SplitHostPort(r.Host)
+	if err != nil {
+		// Common, there often is not port.
+		host = r.Host
+	}
+	dom, err := dns.ParseDomain(host)
+	if err != nil {
+		log().Debugx("parsing http request domain", err, mlog.Field("host", host))
+		http.NotFound(w, r)
+		return true
+	}
+
+	for _, h := range mox.Conf.WebHandlers() {
+		if h.DNSDomain != dom {
+			continue
+		}
+		loc := h.Path.FindStringIndex(r.URL.Path)
+		if loc == nil {
+			continue
+		}
+		s := loc[0]
+		e := loc[1]
+		path := r.URL.Path[s:e]
+
+		if r.TLS == nil && !h.DontRedirectPlainHTTP {
+			u := *r.URL
+			u.Scheme = "https"
+			u.Host = host
+			w.Handler = h.Name
+			http.Redirect(w, r, u.String(), http.StatusPermanentRedirect)
+			return true
+		}
+
+		if h.WebStatic != nil && HandleStatic(h.WebStatic, w, r) {
+			w.Handler = h.Name
+			return true
+		}
+		if h.WebRedirect != nil && HandleRedirect(h.WebRedirect, w, r) {
+			w.Handler = h.Name
+			return true
+		}
+		if h.WebForward != nil && HandleForward(h.WebForward, w, r, path) {
+			w.Handler = h.Name
+			return true
+		}
+	}
+	return false
+}
+
+var lsTemplate = htmltemplate.Must(htmltemplate.New("ls").Parse(`<!doctype html>
+<html>
+	<head>
+		<meta charset="utf-8" />
+		<meta name="viewport" content="width=device-width, initial-scale=1" />
+		<title>ls</title>
+		<style>
+body, html { padding: 1em; font-size: 16px; }
+* { font-size: inherit; font-family: ubuntu, lato, sans-serif; margin: 0; padding: 0; box-sizing: border-box; }
+h1 { margin-bottom: 1ex; font-size: 1.2rem; }
+table td, table th { padding: .2em .5em; }
+table > tbody > tr:nth-child(odd) { background-color: #f8f8f8; }
+[title] { text-decoration: underline; text-decoration-style: dotted; }
+		</style>
+	</head>
+	<body>
+		<h1>ls</h1>
+		<table>
+			<thead>
+				<tr>
+					<th>Size in MB</th>
+					<th>Modified (UTC)</th>
+					<th>Name</th>
+				</tr>
+			</thead>
+			<tbody>
+			{{ if not .Files }}
+				<tr><td colspan="3">No files.</td></tr>
+			{{ end }}
+			{{ range .Files }}
+				<tr>
+					<td title="{{ .Size }} bytes" style="text-align: right">{{ .SizeReadable }}{{ if .SizePad }}<span style="visibility:hidden">.  </span>{{ end }}</td>
+					<td>{{ .Modified }}</td>
+					<td><a style="display: block" href="{{ .Name }}">{{ .Name }}</a></td>
+				</tr>
+			{{ end }}
+			</tbody>
+		</table>
+	</body>
+</html>
+`))
+
+// HandleStatic serves static files. If a directory is requested and the URL
+// path doesn't end with a slash, a response with a redirect to the URL path with trailing
+// slash is written. If a directory is requested and an index.html exists, that
+// file is returned. Otherwise, for directories with ListFiles configured, a
+// directory listing is returned.
+func HandleStatic(h *config.WebStatic, w http.ResponseWriter, r *http.Request) (handled bool) {
+	log := func() *mlog.Log {
+		return xlog.WithContext(r.Context())
+	}
+	recvid := func() string {
+		cid := mox.CidFromCtx(r.Context())
+		if cid <= 0 {
+			return ""
+		}
+		return " (requestid " + mox.ReceivedID(cid) + ")"
+	}
+
+	if r.Method != "GET" && r.Method != "HEAD" {
+		if h.ContinueNotFound {
+			// Give another handler that is presumbly configured, for the same path, a chance.
+			// E.g. an app that may generate this file for future requests to pick up.
+			return false
+		}
+		http.Error(w, "405 - method not allowed", http.StatusMethodNotAllowed)
+		return true
+	}
+
+	var fspath string
+	if h.StripPrefix != "" {
+		if !strings.HasPrefix(r.URL.Path, h.StripPrefix) {
+			if h.ContinueNotFound {
+				// We haven't handled this request, try a next WebHandler in the list.
+				return false
+			}
+			http.NotFound(w, r)
+			return true
+		}
+		fspath = filepath.Join(h.Root, strings.TrimPrefix(r.URL.Path, h.StripPrefix))
+	} else {
+		fspath = filepath.Join(h.Root, r.URL.Path)
+	}
+
+	f, err := os.Open(fspath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			if h.ContinueNotFound {
+				// We haven't handled this request, try a next WebHandler in the list.
+				return false
+			}
+			http.NotFound(w, r)
+			return true
+		} else if os.IsPermission(err) {
+			http.Error(w, "403 - permission denied", http.StatusForbidden)
+			return true
+		}
+		log().Errorx("open file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
+		http.Error(w, "500 - internal server error"+recvid(), http.StatusInternalServerError)
+		return true
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		log().Errorx("stat file for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
+		http.Error(w, "500 - internal server error"+recvid(), http.StatusInternalServerError)
+		return true
+	}
+	if fi.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusTemporaryRedirect)
+		return true
+	}
+
+	serveFile := func(name string, mtime time.Time, content *os.File) {
+		// ServeContent only sets a content-type if not already present in the response headers.
+		hdr := w.Header()
+		for k, v := range h.ResponseHeaders {
+			hdr.Add(k, v)
+		}
+		http.ServeContent(w, r, name, mtime, content)
+	}
+
+	if fi.IsDir() {
+		index, err := os.Open(filepath.Join(fspath, "index.html"))
+		if err != nil && os.IsPermission(err) || err != nil && os.IsNotExist(err) && !h.ListFiles {
+			http.Error(w, "403 - permission denied", http.StatusForbidden)
+			return true
+		} else if err == nil {
+			defer index.Close()
+			var ifi os.FileInfo
+			ifi, err = index.Stat()
+			if err == nil {
+				w.Header().Set("Content-Type", "text/html; charset=utf-8")
+				serveFile(filepath.Base(fspath), ifi.ModTime(), index)
+				return true
+			}
+		}
+		if !os.IsNotExist(err) {
+			log().Errorx("stat for static file serving", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
+			http.Error(w, "500 - internal server error"+recvid(), http.StatusInternalServerError)
+			return true
+		}
+
+		type File struct {
+			Name         string
+			Size         int64
+			SizeReadable string
+			SizePad      bool // Whether the size needs padding because it has no decimal point.
+			Modified     string
+		}
+		files := []File{}
+		if r.URL.Path != "/" {
+			files = append(files, File{"..", 0, "", false, ""})
+		}
+		for {
+			l, err := f.Readdir(1000)
+			for _, e := range l {
+				mb := float64(e.Size()) / (1024 * 1024)
+				var size string
+				var sizepad bool
+				if mb >= 10 {
+					size = fmt.Sprintf("%d", int64(mb))
+					sizepad = true
+				} else {
+					size = fmt.Sprintf("%.2f", mb)
+				}
+				const dateTime = "2006-01-02 15:04:05" // time.DateTime, but only since go1.20.
+				modified := e.ModTime().UTC().Format(dateTime)
+				f := File{e.Name(), e.Size(), size, sizepad, modified}
+				if e.IsDir() {
+					f.Name += "/"
+				}
+				files = append(files, f)
+			}
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				log().Errorx("reading directory for file listing", err, mlog.Field("url", r.URL), mlog.Field("fspath", fspath))
+				http.Error(w, "500 - internal server error"+recvid(), http.StatusInternalServerError)
+				return true
+			}
+		}
+		sort.Slice(files, func(i, j int) bool {
+			return files[i].Name < files[j].Name
+		})
+		hdr := w.Header()
+		hdr.Set("Content-Type", "text/html; charset=utf-8")
+		for k, v := range h.ResponseHeaders {
+			if !strings.EqualFold(k, "content-type") {
+				hdr.Add(k, v)
+			}
+		}
+		err = lsTemplate.Execute(w, map[string]any{"Files": files})
+		if err != nil && !moxio.IsClosed(err) {
+			log().Errorx("executing directory listing template", err)
+		}
+		return true
+	}
+
+	serveFile(fspath, fi.ModTime(), f)
+	return true
+}
+
+// HandleRedirect writes a response with an HTTP redirect.
+func HandleRedirect(h *config.WebRedirect, w http.ResponseWriter, r *http.Request) (handled bool) {
+	var dstpath string
+	if h.OrigPath == nil {
+		// No path rewrite necessary.
+		dstpath = r.URL.Path
+	} else if !h.OrigPath.MatchString(r.URL.Path) {
+		http.NotFound(w, r)
+		return true
+	} else {
+		dstpath = h.OrigPath.ReplaceAllString(r.URL.Path, h.ReplacePath)
+	}
+
+	u := *r.URL
+	u.Opaque = ""
+	u.RawPath = ""
+	u.OmitHost = false
+	if h.URL != nil {
+		u.Scheme = h.URL.Scheme
+		u.Host = h.URL.Host
+		u.ForceQuery = h.URL.ForceQuery
+		u.RawQuery = h.URL.RawQuery
+		u.Fragment = h.URL.Fragment
+	}
+	if r.URL.RawQuery != "" {
+		if u.RawQuery != "" {
+			u.RawQuery += "&"
+		}
+		u.RawQuery += r.URL.RawQuery
+	}
+	u.Path = dstpath
+	code := http.StatusPermanentRedirect
+	if h.StatusCode != 0 {
+		code = h.StatusCode
+	}
+	http.Redirect(w, r, u.String(), code)
+	return true
+}
+
+// HandleForward handles a request by forwarding it to another webserver and
+// passing the response on. I.e. a reverse proxy.
+func HandleForward(h *config.WebForward, w http.ResponseWriter, r *http.Request, path string) (handled bool) {
+	log := func() *mlog.Log {
+		return xlog.WithContext(r.Context())
+	}
+	recvid := func() string {
+		cid := mox.CidFromCtx(r.Context())
+		if cid <= 0 {
+			return ""
+		}
+		return " (requestid " + mox.ReceivedID(cid) + ")"
+	}
+
+	xr := *r
+	r = &xr
+	if h.StripPath {
+		u := *r.URL
+		u.Path = r.URL.Path[len(path):]
+		u.RawPath = ""
+		r.URL = &u
+	}
+
+	// Remove any forwarded headers passed in by client.
+	hdr := http.Header{}
+	for k, vl := range r.Header {
+		switch k {
+		case "Forwarded", "X-Forwarded-For", "X-Forwarded-Host", "X-Forwarded-Proto":
+			continue
+		}
+		hdr[k] = vl
+	}
+	r.Header = hdr
+
+	// Add our own X-Forwarded headers. ReverseProxy will add X-Forwarded-For.
+	r.Header["X-Forwarded-Host"] = []string{r.Host}
+	proto := "http"
+	if r.TLS != nil {
+		proto = "https"
+	}
+	r.Header["X-Forwarded-Proto"] = []string{proto}
+	// todo: add Forwarded header? is anyone using it?
+
+	// ReverseProxy will append any remaining path to the configured target URL.
+	proxy := httputil.NewSingleHostReverseProxy(h.TargetURL)
+	proxy.FlushInterval = time.Duration(-1) // Flush after each write.
+	proxy.ErrorLog = golog.New(mlog.ErrWriter(mlog.New("net/http/httputil").WithContext(r.Context()), mlog.LevelDebug, "reverseproxy error"), "", 0)
+	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
+		log().Errorx("forwarding request to backend webserver", err, mlog.Field("url", r.URL))
+		http.Error(w, "502 - bad gateway"+recvid(), http.StatusBadGateway)
+	}
+	whdr := w.Header()
+	for k, v := range h.ResponseHeaders {
+		whdr.Add(k, v)
+	}
+	proxy.ServeHTTP(w, r)
+	return true
+}

main.go

@@ -105,6 +105,7 @@ var commands = []struct {
 	{"config domain rm", cmdConfigDomainRemove},
 	{"config describe-sendmail", cmdConfigDescribeSendmail},
 	{"config printservice", cmdConfigPrintservice},
+	{"examples", cmdExamples},
 
 	{"checkupdate", cmdCheckupdate},
 	{"cid", cmdCid},
@@ -758,6 +759,109 @@ func cmdConfigDNSCheck(c *cmd) {
 	printResult("Autodiscover", result.Autodiscover.Result)
 }
 
+var examples = []struct {
+	Name string
+	Get  func() string
+}{
+	{
+		"webhandlers",
+		func() string {
+			const webhandlers = `# Snippet of domains.conf to configure WebHandlers.
+WebHandlers:
+	-
+		# The name of the handler, used in logging and metrics.
+		LogName: staticmjl
+		# With ACME configured, each configured domain will automatically get a TLS
+		# certificate on first request.
+		Domain: mox.example
+		PathRegexp: ^/who/mjl/
+		WebStatic:
+			StripPrefix: /who/mjl
+			# Requested path /who/mjl/inferno/ resolves to local web/mjl/inferno.
+			# If a directory contains an index.html, it is served when a directory is requested.
+			Root: web/mjl
+			# With ListFiles true, if a directory does not contain an index.html, the contents are listed.
+			ListFiles: true
+			ResponseHeaders:
+				X-Mox: hi
+	-
+		LogName: redir
+		Domain: mox.example
+		PathRegexp: ^/redir/a/b/c
+		# Don't redirect from plain HTTP to HTTPS.
+		DontRedirectPlainHTTP: true
+		WebRedirect:
+			# Just change the domain and add query string set fragment. No change to scheme.
+			# Path will start with /redir/a/b/c (and whathever came after) because no
+			# OrigPathRegexp+ReplacePath is set.
+			BaseURL: //moxest.example?q=1#frag
+			# Default redirection is 308 - Permanent Redirect.
+			StatusCode: 307
+	-
+		LogName: oldnew
+		Domain: mox.example
+		PathRegexp: ^/old/
+		WebRedirect:
+			# Replace path, leaving rest of URL intact.
+			OrigPathRegexp: ^/old/(.*)
+			ReplacePath: /new/$1
+	-
+		LogName: app
+		Domain: mox.example
+		PathRegexp: ^/app/
+		WebForward:
+			# Strip the path matched by PathRegexp before forwarding the request. So original
+			# request /app/api become just /api.
+			StripPath: true
+			# URL of backend, where requests are forwarded to. The path in the URL is kept,
+			# so for incoming request URL /app/api, the outgoing request URL has path /app-v2/api.
+			# Requests are made with Go's net/http DefaultTransporter, including using
+			# HTTP_PROXY and HTTPS_PROXY environment variables.
+			URL: http://127.0.0.1:8900/app-v2/
+			# Add headers to response.
+			ResponseHeaders:
+				X-Frame-Options: deny
+				X-Content-Type-Options: nosniff
+`
+			// Parse just so we know we have the syntax right.
+			// todo: ideally we would have a complete config file and parse it fully.
+			var conf struct {
+				WebHandlers []config.WebHandler
+			}
+			err := sconf.Parse(strings.NewReader(webhandlers), &conf)
+			xcheckf(err, "parsing webhandlers example")
+			return webhandlers
+		},
+	},
+}
+
+func cmdExamples(c *cmd) {
+	c.params = "[name]"
+	c.help = `List available examples, or print a specific example.`
+
+	args := c.Parse()
+	if len(args) > 1 {
+		c.Usage()
+	}
+
+	var match func() string
+	for _, ex := range examples {
+		if len(args) == 0 {
+			fmt.Println(ex.Name)
+		} else if args[0] == ex.Name {
+			match = ex.Get
+		}
+	}
+	if len(args) == 0 {
+		return
+	}
+	if match == nil {
+		log.Fatalln("not found")
+	}
+	fmt.Print(match())
+	return
+}
+
 func cmdLoglevels(c *cmd) {
 	c.params = "[level [pkg]]"
 	c.help = `Print the log levels, or set a new default log level, or a level for the given package.

mox-/cid.go

@@ -1,8 +1,11 @@
 package mox
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
+
+	"github.com/mjl-/mox/mlog"
 )
 
 var cid atomic.Int64
@@ -15,3 +18,12 @@ func init() {
 func Cid() int64 {
 	return cid.Add(1)
 }
+
+// CidFromCtx returns the cid in the context, or 0.
+func CidFromCtx(ctx context.Context) int64 {
+	v := ctx.Value(mlog.CidKey)
+	if v == nil {
+		return 0
+	}
+	return v.(int64)
+}

mox-/config.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/url"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -138,6 +139,7 @@ func (c *Config) loadDynamic() []error {
 	c.Dynamic = d
 	c.dynamicMtime = mtime
 	c.accountDestinations = accDests
+	c.allowACMEHosts()
 	return nil
 }
 
@@ -196,25 +198,39 @@ func (c *Config) AccountDestination(addr string) (accDests AccountDestination, o
 	return
 }
 
+func (c *Config) WebHandlers() (l []config.WebHandler) {
+	c.withDynamicLock(func() {
+		l = c.Dynamic.WebHandlers
+	})
+	return l
+}
+
 func (c *Config) allowACMEHosts() {
-	// todo future: reset the allowed hosts for autoconfig & mtasts when loading new list.
 	for _, l := range c.Static.Listeners {
 		if l.TLS == nil || l.TLS.ACME == "" {
 			continue
 		}
+
 		m := c.Static.ACME[l.TLS.ACME].Manager
+		hostnames := map[dns.Domain]struct{}{}
+
+		hostnames[c.Static.HostnameDomain] = struct{}{}
+		if l.HostnameDomain.ASCII != "" {
+			hostnames[l.HostnameDomain] = struct{}{}
+		}
+
 		for _, dom := range c.Dynamic.Domains {
 			if l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS {
 				if d, err := dns.ParseDomain("autoconfig." + dom.Domain.ASCII); err != nil {
 					xlog.Errorx("parsing autoconfig domain", err, mlog.Field("domain", dom.Domain))
 				} else {
-					m.AllowHostname(d)
+					hostnames[d] = struct{}{}
 				}
 
 				if d, err := dns.ParseDomain("autodiscover." + dom.Domain.ASCII); err != nil {
 					xlog.Errorx("parsing autodiscover domain", err, mlog.Field("domain", dom.Domain))
 				} else {
-					m.AllowHostname(d)
+					hostnames[d] = struct{}{}
 				}
 			}
 
@@ -222,11 +238,19 @@ func (c *Config) allowACMEHosts() {
 				d, err := dns.ParseDomain("mta-sts." + dom.Domain.ASCII)
 				if err != nil {
 					xlog.Errorx("parsing mta-sts domain", err, mlog.Field("domain", dom.Domain))
-					continue
+				} else {
+					hostnames[d] = struct{}{}
 				}
-				m.AllowHostname(d)
 			}
 		}
+
+		if l.WebserverHTTPS.Enabled {
+			for _, wh := range c.Dynamic.WebHandlers {
+				hostnames[wh.DNSDomain] = struct{}{}
+			}
+		}
+
+		m.SetAllowedHostnames(hostnames)
 	}
 }
 
@@ -529,6 +553,7 @@ func PrepareStaticConfig(ctx context.Context, configFile string, config *Config,
 			needtls("AdminHTTPS", l.AdminHTTPS.Enabled)
 			needtls("AutoconfigHTTPS", l.AutoconfigHTTPS.Enabled && !l.AutoconfigHTTPS.NonTLS)
 			needtls("MTASTSHTTPS", l.MTASTSHTTPS.Enabled && !l.MTASTSHTTPS.NonTLS)
+			needtls("WebserverHTTPS", l.WebserverHTTPS.Enabled)
 			if len(needsTLS) > 0 {
 				addErrorf("listener %q does not specify tls config, but requires tls for %s", name, strings.Join(needsTLS, ", "))
 			}
@@ -964,6 +989,100 @@ func prepareDynamicConfig(ctx context.Context, dynamicPath string, static config
 		checkMailboxNormf(tlsrpt.Mailbox, "TLSRPT mailbox for account %q", tlsrpt.Account)
 		accDests[addrFull] = AccountDestination{lp, tlsrpt.Account, dest}
 	}
+
+	// Check webserver configs.
+	for i := range c.WebHandlers {
+		wh := &c.WebHandlers[i]
+
+		if wh.LogName == "" {
+			wh.Name = fmt.Sprintf("%d", i)
+		} else {
+			wh.Name = wh.LogName
+		}
+
+		dom, err := dns.ParseDomain(wh.Domain)
+		if err != nil {
+			addErrorf("webhandler %s %s: parsing domain: %v", wh.Domain, wh.PathRegexp, err)
+		}
+		wh.DNSDomain = dom
+
+		if !strings.HasPrefix(wh.PathRegexp, "^") {
+			addErrorf("webhandler %s %s: path regexp must start with a ^", wh.Domain, wh.PathRegexp)
+		}
+		re, err := regexp.Compile(wh.PathRegexp)
+		if err != nil {
+			addErrorf("webhandler %s %s: compiling regexp: %v", wh.Domain, wh.PathRegexp, err)
+		}
+		wh.Path = re
+
+		var n int
+		if wh.WebStatic != nil {
+			n++
+			ws := wh.WebStatic
+			if ws.StripPrefix != "" && !strings.HasPrefix(ws.StripPrefix, "/") {
+				addErrorf("webstatic %s %s: prefix to strip %s must start with a slash", wh.Domain, wh.PathRegexp, ws.StripPrefix)
+			}
+			for k := range ws.ResponseHeaders {
+				xk := k
+				k := strings.TrimSpace(xk)
+				if k != xk || k == "" {
+					addErrorf("webstatic %s %s: bad header %q", wh.Domain, wh.PathRegexp, xk)
+				}
+			}
+		}
+		if wh.WebRedirect != nil {
+			n++
+			wr := wh.WebRedirect
+			if wr.BaseURL != "" {
+				u, err := url.Parse(wr.BaseURL)
+				if err != nil {
+					addErrorf("webredirect %s %s: parsing redirect url %s: %v", wh.Domain, wh.PathRegexp, wr.BaseURL, err)
+				}
+				switch u.Path {
+				case "", "/":
+					u.Path = "/"
+				default:
+					addErrorf("webredirect %s %s: BaseURL must have empty path", wh.Domain, wh.PathRegexp, wr.BaseURL)
+				}
+				wr.URL = u
+			}
+			if wr.OrigPathRegexp != "" && wr.ReplacePath != "" {
+				re, err := regexp.Compile(wr.OrigPathRegexp)
+				if err != nil {
+					addErrorf("webredirect %s %s: compiling regexp %s: %v", wh.Domain, wh.PathRegexp, wr.OrigPathRegexp, err)
+				}
+				wr.OrigPath = re
+			} else if wr.OrigPathRegexp != "" || wr.ReplacePath != "" {
+				addErrorf("webredirect %s %s: must have either both OrigPathRegexp and ReplacePath, or neither", wh.Domain, wh.PathRegexp)
+			} else if wr.BaseURL == "" {
+				addErrorf("webredirect %s %s: must at least one of BaseURL and OrigPathRegexp+ReplacePath", wh.Domain, wh.PathRegexp)
+			}
+			if wr.StatusCode != 0 && (wr.StatusCode < 300 || wr.StatusCode >= 400) {
+				addErrorf("webredirect %s %s: invalid redirect status code %d", wh.Domain, wh.PathRegexp, wr.StatusCode)
+			}
+		}
+		if wh.WebForward != nil {
+			n++
+			wf := wh.WebForward
+			u, err := url.Parse(wf.URL)
+			if err != nil {
+				addErrorf("webforward %s %s: parsing url %s: %v", wh.Domain, wh.PathRegexp, wf.URL, err)
+			}
+			wf.TargetURL = u
+
+			for k := range wf.ResponseHeaders {
+				xk := k
+				k := strings.TrimSpace(xk)
+				if k != xk || k == "" {
+					addErrorf("webforward %s %s: bad header %q", wh.Domain, wh.PathRegexp, xk)
+				}
+			}
+		}
+		if n != 1 {
+			addErrorf("webhandler %s %s: must have exactly one handler, not %d", wh.Domain, wh.PathRegexp, n)
+		}
+	}
+
 	return
 }
 

serve.go

@@ -139,9 +139,14 @@ requested, other TLS certificates are requested on demand.
 	if len(args) != 0 {
 		c.Usage()
 	}
+
+	// Set debug logging until config is fully loaded.
+	mlog.Logfmt = true
+	mox.Conf.Log[""] = mlog.LevelDebug
+	mlog.SetConfig(mox.Conf.Log)
+
 	mox.MustLoadConfig()
 
-	mlog.Logfmt = true
 	log := mlog.New("serve")
 
 	if os.Getuid() == 0 {