implement transparent gzip compression in the webserver

we only compress if applicable (content-type indicates likely compressible), client supports it, response doesn't already have a content-encoding). for internal handlers, we always enable compression. for reverse proxied and static files, compression must be enabled per handler. for internal & reverse proxy handlers, we do streaming compression at "bestspeed" quality (probably level 1). for static files, we have a cache based on mtime with fixed max size, where we evict based on least recently used. we compress with the default level (more cpu, better ratio).
2024-12-26 16:33:47 +03:00 · 2023-08-21 21:52:35 +02:00 · 2023-08-21 21:52:35 +02:00 · 9e248860ee
commit 9e248860ee
parent 4c72184b44
13 changed files with 696 additions and 33 deletions
--- a/config/config.go
+++ b/config/config.go
@ -428,6 +428,7 @@ type WebHandler struct {
 	Domain                string       `sconf-doc:"Both Domain and PathRegexp must match for this WebHandler to match a request. Exactly one of WebStatic, WebRedirect, WebForward must be set."`
 	PathRegexp            string       `sconf-doc:"Regular expression matched against request path, must always start with ^ to ensure matching from the start of the path. The matching prefix can optionally be stripped by WebForward. The regular expression does not have to end with $."`
 	DontRedirectPlainHTTP bool         `sconf:"optional" sconf-doc:"If set, plain HTTP requests are not automatically permanently redirected (308) to HTTPS. If you don't have a HTTPS webserver configured, set this to true."`
+	Compress              bool         `sconf:"optional" sconf-doc:"Transparently compress responses (currently with gzip) if the client supports it, the status is 200 OK, no Content-Encoding is set on the response yet and the Content-Type of the response hints that the data is compressible (text/..., specific application/... and .../...+json and .../...+xml). For static files only, a cache with compressed files is kept."`
 	WebStatic             *WebStatic   `sconf:"optional" sconf-doc:"Serve static files."`
 	WebRedirect           *WebRedirect `sconf:"optional" sconf-doc:"Redirect requests to configured URL."`
 	WebForward            *WebForward  `sconf:"optional" sconf-doc:"Forward requests to another webserver, i.e. reverse proxy."`
--- a/config/doc.go
+++ b/config/doc.go
@ -910,6 +910,13 @@ describe-static" and "mox config describe-domains":
 			# (optional)
 			DontRedirectPlainHTTP: false

+			# Transparently compress responses (currently with gzip) if the client supports
+			# it, the status is 200 OK, no Content-Encoding is set on the response yet and the
+			# Content-Type of the response hints that the data is compressible (text/...,
+			# specific application/... and .../...+json and .../...+xml). For static files
+			# only, a cache with compressed files is kept. (optional)
+			Compress: false
+
 			# Serve static files. (optional)
 			WebStatic:

--- a/http/atime.go
+++ b/http/atime.go
@ -0,0 +1,9 @@
+//go:build !netbsd && !freebsd && !darwin
+
+package http
+
+import "syscall"
+
+func statAtime(sys *syscall.Stat_t) int64 {
+	return int64(sys.Atim.Sec)*1000*1000*1000 + int64(sys.Atim.Nsec)
+}
--- a/http/atime_bsd.go
+++ b/http/atime_bsd.go
@ -0,0 +1,9 @@
+//go:build netbsd || freebsd || darwin
+
+package http
+
+import "syscall"
+
+func statAtime(sys *syscall.Stat_t) int64 {
+	return int64(sys.Atimespec.Sec)*1000*1000*1000 + int64(sys.Atimespec.Nsec)
+}
--- a/http/gzcache.go
+++ b/http/gzcache.go
@ -0,0 +1,430 @@
+package http
+
+import (
+	"compress/gzip"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+// todo: consider caching gzipped responses from forward handlers too. we would need to read the responses (handle up to perhaps 2mb), hash the data (blake2b seems fast), check if we have the gzip content for that hash, cache it on second request. keep around entries for non-yet-cached hashes, with some limit and lru eviction policy. we have to recognize some content-types as not applicable and do direct streaming compression, e.g. for text/event-stream. and we need to detect when backend server could be slowly sending out data and abort the caching attempt. downside is always that we need to read the whole response before and hash it before we can send our response. it is best if the backend just responds with gzip itself though. compression needs more cpu than hashing (at least 10x), but it's only worth it with enough hits.
+
+// Cache for gzipped static files.
+var staticgzcache gzcache
+
+type gzcache struct {
+	dir string // Where all files are stored.
+
+	// Max total size of combined files in cache. When adding a new entry, the least
+	// recently used entries are evicted to stay below this size.
+	maxSize int64
+
+	sync.Mutex
+
+	// Total on-disk size of compressed data. Not larger than maxSize. We can
+	// temporarily have more bytes in use because while/after evicting, a writer may
+	// still have the old removed file open.
+	size int64
+
+	// Indexed by effective path, based on handler.
+	paths map[string]gzfile
+
+	// Only with files we completed compressing, kept ordered by atime. We evict from
+	// oldest. On use, we take entries out and put them at newest.
+	oldest, newest *pathUse
+}
+
+type gzfile struct {
+	// Whether compressing in progress. If a new request comes in while we are already
+	// compressing, for simplicity of code we just compress again for that client.
+	compressing bool
+
+	mtime  int64    // If mtime changes, we remove entry from cache.
+	atime  int64    // For LRU.
+	gzsize int64    // Compressed size, used in Content-Length header.
+	use    *pathUse // Only set after compressing finished.
+}
+
+type pathUse struct {
+	prev, next *pathUse // Double-linked list.
+	path       string
+}
+
+// Initialize staticgzcache from on-disk directory.
+// The path and mtime are in the filename, the atime is in the file itself.
+func loadStaticGzipCache(dir string, maxSize int64) {
+	staticgzcache = gzcache{
+		dir:     dir,
+		maxSize: maxSize,
+		paths:   map[string]gzfile{},
+	}
+
+	// todo future: should we split cached files in sub directories, so we don't end up with one huge directory?
+	os.MkdirAll(dir, 0700)
+	entries, err := os.ReadDir(dir)
+	if err != nil && !os.IsNotExist(err) {
+		xlog.Errorx("listing static gzip cache files", err, mlog.Field("dir", dir))
+	}
+	for _, e := range entries {
+		name := e.Name()
+		var err error
+		if !strings.HasSuffix(name, ".gz") {
+			err = errors.New("missing .gz suffix")
+		}
+		var path, xpath, mtimestr string
+		if err == nil {
+			var ok bool
+			xpath, mtimestr, ok = strings.Cut(strings.TrimRight(name, ".gz"), "+")
+			if !ok {
+				err = fmt.Errorf("missing + in filename")
+			}
+		}
+		if err == nil {
+			var pathbuf []byte
+			pathbuf, err = base64.RawURLEncoding.DecodeString(xpath)
+			if err == nil {
+				path = string(pathbuf)
+			}
+		}
+		var mtime int64
+		if err == nil {
+			mtime, err = strconv.ParseInt(mtimestr, 16, 64)
+		}
+		var fi fs.FileInfo
+		if err == nil {
+			fi, err = e.Info()
+		}
+		var atime int64
+		if err == nil {
+			if sys, sysok := fi.Sys().(*syscall.Stat_t); !sysok {
+				err = fmt.Errorf("FileInfo.Sys not a *syscall.Stat_t but %T", fi.Sys())
+			} else {
+				atime = statAtime(sys)
+			}
+		}
+		if err != nil {
+			xlog.Infox("removing unusable/unrecognized file in static gzip cache dir", err)
+			xerr := os.Remove(filepath.Join(dir, name))
+			xlog.Check(xerr, "removing unusable file in static gzip cache dir", mlog.Field("error", err), mlog.Field("dir", dir), mlog.Field("filename", name))
+			continue
+		}
+		staticgzcache.paths[path] = gzfile{
+			mtime:  mtime,
+			atime:  atime,
+			gzsize: fi.Size(),
+			use:    &pathUse{path: path},
+		}
+		staticgzcache.size += fi.Size()
+	}
+
+	pathatimes := make([]struct {
+		path  string
+		atime int64
+	}, len(staticgzcache.paths))
+	i := 0
+	for k, gf := range staticgzcache.paths {
+		pathatimes[i].path = k
+		pathatimes[i].atime = gf.atime
+		i++
+	}
+	sort.Slice(pathatimes, func(i, j int) bool {
+		return pathatimes[i].atime < pathatimes[j].atime
+	})
+	for _, pa := range pathatimes {
+		staticgzcache.push(staticgzcache.paths[pa.path].use)
+	}
+
+	// Ensure cache size is OK for current config.
+	staticgzcache.evictFor(0)
+}
+
+// Evict entries so size bytes are available.
+// Must be called with lock held.
+func (c *gzcache) evictFor(size int64) {
+	for c.size+size > c.maxSize && c.oldest != nil {
+		c.evictPath(c.oldest.path)
+	}
+}
+
+// remove path from cache.
+// Must be called with lock held.
+func (c *gzcache) evictPath(path string) {
+	gf := c.paths[path]
+
+	delete(c.paths, path)
+	c.unlink(gf.use)
+	c.size -= gf.gzsize
+	err := os.Remove(staticCachePath(c.dir, path, gf.mtime))
+	xlog.Check(err, "removing cached gzipped static file", mlog.Field("path", path))
+}
+
+// Open cached file for path, requiring it has mtime. If there is no usable cached
+// file, a nil file is returned and the caller should compress and add to the cache
+// with startPath and finishPath. No usable cached file means the path isn't in the
+// cache, or its mtime is different, or there is an entry but it is new and being
+// compressed at the moment. If a usable cached file was found, it is opened and
+// returned, along with its compressed/on-disk size.
+func (c *gzcache) openPath(path string, mtime int64) (*os.File, int64) {
+	c.Lock()
+	defer c.Unlock()
+
+	gf, ok := c.paths[path]
+	if !ok || gf.compressing {
+		return nil, 0
+	}
+	if gf.mtime != mtime {
+		// File has changed, remove old entry. Caller will add to cache again.
+		c.evictPath(path)
+		return nil, 0
+	}
+
+	p := staticCachePath(c.dir, path, gf.mtime)
+	f, err := os.Open(p)
+	if err != nil {
+		xlog.Errorx("open static cached gzip file, removing from cache", err, mlog.Field("path", path))
+		// Perhaps someone removed the file? Remove from cache, it will be recreated.
+		c.evictPath(path)
+		return nil, 0
+	}
+
+	gf.atime = time.Now().UnixNano()
+	c.unlink(gf.use)
+	c.push(gf.use)
+	c.paths[path] = gf
+
+	return f, gf.gzsize
+}
+
+// startPath attempts to add an entry to the cache for a new cached compressed
+// file. If there is already an entry but it isn't done compressing yet, false is
+// returned and the caller can still compress and respond but the entry cannot be
+// added to the cache. If the entry is being added, the caller must call finishPath
+// or abortPath.
+func (c *gzcache) startPath(path string, mtime int64) bool {
+	c.Lock()
+	defer c.Unlock()
+
+	if _, ok := c.paths[path]; ok {
+		return false
+	}
+	// note: no "use" yet, we only set that when we finish, so we don't have to clean up on abort.
+	c.paths[path] = gzfile{compressing: true, mtime: mtime}
+	return true
+}
+
+// finishPath completes adding an entry to the cache, marking the entry as
+// compressed, accounting for its size, and marking its atime.
+func (c *gzcache) finishPath(path string, gzsize int64) {
+	c.Lock()
+	defer c.Unlock()
+
+	c.evictFor(gzsize)
+
+	gf := c.paths[path]
+	gf.compressing = false
+	gf.gzsize = gzsize
+	gf.atime = time.Now().UnixNano()
+	gf.use = &pathUse{path: path}
+	c.paths[path] = gf
+	c.size += gzsize
+	c.push(gf.use)
+}
+
+// abortPath marks an entry as no longer being added to the cache.
+func (c *gzcache) abortPath(path string) {
+	c.Lock()
+	defer c.Unlock()
+
+	delete(c.paths, path)
+	// note: gzfile.use isn't set yet.
+}
+
+// push inserts the "pathUse" to the head of the LRU doubly-linked list, unlinking
+// it first if needed.
+func (c *gzcache) push(u *pathUse) {
+	c.unlink(u)
+	u.prev = c.newest
+	if c.newest != nil {
+		c.newest.next = u
+	}
+	if c.oldest == nil {
+		c.oldest = u
+	}
+	c.newest = u
+}
+
+// unlink removes the "pathUse" from the LRU doubly-linked list.
+func (c *gzcache) unlink(u *pathUse) {
+	if c.oldest == u {
+		c.oldest = u.next
+	}
+	if c.newest == u {
+		c.newest = u.prev
+	}
+	if u.prev != nil {
+		u.prev.next = u.next
+	}
+	if u.next != nil {
+		u.next.prev = u.prev
+	}
+	u.prev = nil
+	u.next = nil
+}
+
+// Return path to the on-disk gzipped cached file.
+func staticCachePath(dir, path string, mtime int64) string {
+	p := base64.RawURLEncoding.EncodeToString([]byte(path))
+	return filepath.Join(dir, fmt.Sprintf("%s+%x.gz", p, mtime))
+}
+
+// staticgzcacheReplacer intercepts responses for cacheable static files,
+// responding with the cached content if appropriate and failing further writes so
+// the regular response writer stops.
+type staticgzcacheReplacer struct {
+	w            http.ResponseWriter
+	r            *http.Request // For its context, or logging.
+	uncomprPath  string
+	uncomprFile  *os.File
+	uncomprMtime time.Time
+	uncomprSize  int64
+
+	statusCode int
+
+	// Set during WriteHeader to indicate a compressed file has been written, further
+	// Writes result in an error to stop the writer of the uncompressed content.
+	handled bool
+}
+
+func (w *staticgzcacheReplacer) logger() *mlog.Log {
+	return xlog.WithContext(w.r.Context())
+}
+
+// Header returns the header of the underlying ResponseWriter.
+func (w *staticgzcacheReplacer) Header() http.Header {
+	return w.w.Header()
+}
+
+// WriteHeader checks whether the response is eligable for compressing. If not,
+// WriteHeader on the underlying ResponseWriter is called. If so, headers for gzip
+// content are set and the gzip content is written, either from disk or compressed
+// and stored in the cache.
+func (w *staticgzcacheReplacer) WriteHeader(statusCode int) {
+	if w.statusCode != 0 {
+		return
+	}
+	w.statusCode = statusCode
+	if statusCode != http.StatusOK {
+		w.w.WriteHeader(statusCode)
+		return
+	}
+
+	gzf, gzsize := staticgzcache.openPath(w.uncomprPath, w.uncomprMtime.UnixNano())
+	if gzf == nil {
+		// Not in cache, or work in progress.
+		started := staticgzcache.startPath(w.uncomprPath, w.uncomprMtime.UnixNano())
+		if !started {
+			// Another request is already compressing and storing this file.
+			// todo: we should just wait for the other compression to finish, then use its result.
+			w.w.(*loggingWriter).UncompressedSize = w.uncomprSize
+			h := w.w.Header()
+			h.Set("Content-Encoding", "gzip")
+			h.Del("Content-Length") // We don't know this, we compress streamingly.
+			gzw, _ := gzip.NewWriterLevel(w.w, gzip.BestSpeed)
+			_, err := io.Copy(gzw, w.uncomprFile)
+			if err == nil {
+				err = gzw.Close()
+			}
+			w.handled = true
+			if err != nil {
+				w.w.(*loggingWriter).error(err)
+			}
+			return
+		}
+
+		// Compress and write to cache.
+		p := staticCachePath(staticgzcache.dir, w.uncomprPath, w.uncomprMtime.UnixNano())
+		ngzf, err := os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
+		if err != nil {
+			w.logger().Errorx("create new static gzip cache file", err, mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p))
+			staticgzcache.abortPath(w.uncomprPath)
+			return
+		}
+		defer func() {
+			if ngzf != nil {
+				staticgzcache.abortPath(w.uncomprPath)
+				err := ngzf.Close()
+				w.logger().Check(err, "closing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p))
+				err = os.Remove(p)
+				w.logger().Check(err, "removing failed static gzip cache file", mlog.Field("requestpath", w.uncomprPath), mlog.Field("fspath", p))
+			}
+		}()
+
+		gzw := gzip.NewWriter(ngzf)
+		_, err = io.Copy(gzw, w.uncomprFile)
+		if err == nil {
+			err = gzw.Close()
+		}
+		if err == nil {
+			err = ngzf.Sync()
+		}
+		if err == nil {
+			gzsize, err = ngzf.Seek(0, 1)
+		}
+		if err == nil {
+			_, err = ngzf.Seek(0, 0)
+		}
+		if err != nil {
+			w.w.(*loggingWriter).error(err)
+			return
+		}
+		staticgzcache.finishPath(w.uncomprPath, gzsize)
+		gzf = ngzf
+		ngzf = nil
+	}
+	defer func() {
+		if gzf != nil {
+			err := gzf.Close()
+			if err != nil {
+				w.logger().Errorx("closing static gzip cache file", err)
+			}
+		}
+	}()
+
+	// Signal to Write that we aleady (attempted to) write the responses.
+	w.handled = true
+
+	w.w.(*loggingWriter).UncompressedSize = w.uncomprSize
+	h := w.w.Header()
+	h.Set("Content-Encoding", "gzip")
+	h.Set("Content-Length", fmt.Sprintf("%d", gzsize))
+	w.w.WriteHeader(statusCode)
+	if _, err := io.Copy(w.w, gzf); err != nil {
+		w.w.(*loggingWriter).error(err)
+	}
+}
+
+var errHandledCompressed = errors.New("response written with compression")
+
+func (w *staticgzcacheReplacer) Write(buf []byte) (int, error) {
+	if w.statusCode == 0 {
+		w.WriteHeader(http.StatusOK)
+	}
+	if w.handled {
+		// For 200 OK, we already wrote the response and just want the caller to stop processing.
+		return 0, errHandledCompressed
+	}
+	return w.w.Write(buf)
+}
--- a/http/web.go
+++ b/http/web.go
@ -4,9 +4,11 @@
 package http

 import (
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io"
 	golog "log"
 	"net"
 	"net/http"
@ -70,8 +72,6 @@ var (
 	)
 )

-// todo: automatic gzip on responses, if client supports it, content is not already compressed. in case of static file only if it isn't too large. skip for certain response content-types (image/*, video/*), or file extensions if there is no identifying content-type. if cpu load isn't too high. if first N kb look compressible and come in quickly enough after first byte (e.g. within 100ms). always flush after 100ms to prevent stalled real-time connections.
-
 type responseWriterFlusher interface {
 	http.ResponseWriter
 	http.Flusher
@ -84,11 +84,15 @@ type loggingWriter struct {
 	R                *http.Request
 	WebsocketRequest bool // Whether request from was websocket.

-	Handler string // Set by router.
+	// Set by router.
+	Handler  string
+	Compress bool

 	// Set by handlers.
 	StatusCode                   int
-	Size                         int64 // Of data served, for non-websocket responses.
+	Size                         int64        // Of data served to client, for non-websocket responses.
+	UncompressedSize             int64        // Can be set by a handler that already serves compressed data, and we update it while compressing.
+	Gzip                         *gzip.Writer // Only set if we transparently compress within loggingWriter (static handlers handle compression themselves, with a cache).
 	Err                          error
 	WebsocketResponse            bool        // If this was a successful websocket connection with backend.
 	SizeFromClient, SizeToClient int64       // Websocket data.
@ -119,6 +123,36 @@ func (w *loggingWriter) proto(websocket bool) string {
 	return proto
 }

+func (w *loggingWriter) Write(buf []byte) (int, error) {
+	if w.StatusCode == 0 {
+		w.WriteHeader(http.StatusOK)
+	}
+
+	var n int
+	var err error
+	if w.Gzip == nil {
+		n, err = w.W.Write(buf)
+		if n > 0 {
+			w.Size += int64(n)
+		}
+	} else {
+		// We flush after each write. Probably takes a few more bytes, but prevents any
+		// issues due to buffering.
+		// w.Gzip.Write updates w.Size with the compressed byte count.
+		n, err = w.Gzip.Write(buf)
+		if err != nil {
+			err = w.Gzip.Flush()
+		}
+		if n > 0 {
+			w.UncompressedSize += int64(n)
+		}
+	}
+	if err != nil {
+		w.error(err)
+	}
+	return n, err
+}
+
 func (w *loggingWriter) setStatusCode(statusCode int) {
 	if w.StatusCode != 0 {
 		return
@ -129,26 +163,108 @@ func (w *loggingWriter) setStatusCode(statusCode int) {
 	metricRequest.WithLabelValues(w.Handler, w.proto(w.WebsocketRequest), method, fmt.Sprintf("%d", w.StatusCode)).Observe(float64(time.Since(w.Start)) / float64(time.Second))
 }

-func (w *loggingWriter) Write(buf []byte) (int, error) {
-	if w.Size == 0 {
-		w.setStatusCode(http.StatusOK)
-	}
-
-	n, err := w.W.Write(buf)
-	if n > 0 {
-		w.Size += int64(n)
-	}
-	if err != nil {
-		w.error(err)
-	}
-	return n, err
+// SetUncompressedSize is used through an interface by
+// ../webmail/webmail.go:/WriteHeader, preventing an import cycle.
+func (w *loggingWriter) SetUncompressedSize(origSize int64) {
+	w.UncompressedSize = origSize
 }

 func (w *loggingWriter) WriteHeader(statusCode int) {
+	if w.StatusCode != 0 {
+		return
+	}
+
 	w.setStatusCode(statusCode)
+
+	// We transparently gzip-compress responses for requests under these conditions, all must apply:
+	//
+	// - Enabled for handler (static handlers make their own decisions).
+	// - Not a websocket request.
+	// - Regular success responses (not errors, or partial content or redirects or "not modified", etc).
+	// - Not already compressed, or any other Content-Encoding header (including "identity").
+	// - Client accepts gzip encoded responses.
+	// - The response has a content-type that is compressible (text/*, */*+{json,xml}, and a few common files (e.g. json, xml, javascript).
+	if w.Compress && !w.WebsocketRequest && statusCode == http.StatusOK && w.W.Header().Values("Content-Encoding") == nil && acceptsGzip(w.R) && compressibleContentType(w.W.Header().Get("Content-Type")) {
+		// todo: we should gather the first kb of data, see if it is compressible. if not, just return original. should set timer so we flush if it takes too long to gather 1kb. for smaller data we shouldn't compress at all.
+
+		// We track the gzipped output for the access log.
+		cw := countWriter{Writer: w.W, Size: &w.Size}
+		w.Gzip, _ = gzip.NewWriterLevel(cw, gzip.BestSpeed)
+		w.W.Header().Set("Content-Encoding", "gzip")
+		w.W.Header().Del("Content-Length") // No longer valid, set again for small responses by net/http.
+	}
 	w.W.WriteHeader(statusCode)
 }

+func acceptsGzip(r *http.Request) bool {
+	s := r.Header.Get("Accept-Encoding")
+	t := strings.Split(s, ",")
+	for _, e := range t {
+		e = strings.TrimSpace(e)
+		tt := strings.Split(e, ";")
+		if len(tt) > 1 && t[1] == "q=0" {
+			continue
+		}
+		if tt[0] == "gzip" {
+			return true
+		}
+	}
+	return false
+}
+
+var compressibleTypes = map[string]bool{
+	"application/csv":          true,
+	"application/javascript":   true,
+	"application/json":         true,
+	"application/x-javascript": true,
+	"application/xml":          true,
+	"image/vnd.microsoft.icon": true,
+	"image/x-icon":             true,
+	"font/ttf":                 true,
+	"font/eot":                 true,
+	"font/otf":                 true,
+	"font/opentype":            true,
+}
+
+func compressibleContentType(ct string) bool {
+	ct = strings.SplitN(ct, ";", 2)[0]
+	ct = strings.TrimSpace(ct)
+	ct = strings.ToLower(ct)
+	if compressibleTypes[ct] {
+		return true
+	}
+	t, st, _ := strings.Cut(ct, "/")
+	return t == "text" || strings.HasSuffix(st, "+json") || strings.HasSuffix(st, "+xml")
+}
+
+func compressibleContent(f *os.File) bool {
+	// We don't want to store many small files. They take up too much disk overhead.
+	if fi, err := f.Stat(); err != nil || fi.Size() < 1024 || fi.Size() > 10*1024*1024 {
+		return false
+	}
+
+	buf := make([]byte, 512)
+	n, err := f.ReadAt(buf, 0)
+	if err != nil && err != io.EOF {
+		return false
+	}
+	ct := http.DetectContentType(buf[:n])
+	return compressibleContentType(ct)
+}
+
+type countWriter struct {
+	Writer io.Writer
+	Size   *int64
+}
+
+func (w countWriter) Write(buf []byte) (int, error) {
+	n, err := w.Writer.Write(buf)
+	if n > 0 {
+		*w.Size += int64(n)
+	}
+	return n, err
+}
+
 var tlsVersions = map[uint16]string{
 	tls.VersionTLS10: "tls1.0",
 	tls.VersionTLS11: "tls1.1",
@ -173,6 +289,12 @@ func (w *loggingWriter) error(err error) {
 }

 func (w *loggingWriter) Done() {
+	if w.Err == nil && w.Gzip != nil {
+		if err := w.Gzip.Close(); err != nil {
+			w.error(err)
+		}
+	}
+
 	method := metricHTTPMethod(w.R.Method)
 	metricResponse.WithLabelValues(w.Handler, w.proto(w.WebsocketResponse), method, fmt.Sprintf("%d", w.StatusCode)).Observe(float64(time.Since(w.Start)) / float64(time.Second))

@ -213,6 +335,11 @@ func (w *loggingWriter) Done() {
 			mlog.Field("sizetoclient", w.SizeToClient),
 			mlog.Field("sizefromclient", w.SizeFromClient),
 		)
+	} else if w.UncompressedSize > 0 {
+		fields = append(fields,
+			mlog.Field("size", w.Size),
+			mlog.Field("uncompressedsize", w.UncompressedSize),
+		)
 	} else {
 		fields = append(fields,
 			mlog.Field("size", w.Size),
@ -338,6 +465,7 @@ func (s *serve) ServeHTTP(xw http.ResponseWriter, r *http.Request) {
 		}
 		if r.URL.Path == h.Path || strings.HasSuffix(h.Path, "/") && strings.HasPrefix(r.URL.Path, h.Path) {
 			nw.Handler = h.Name
+			nw.Compress = true
 			h.Handler.ServeHTTP(nw, r)
 			return
 		}
@ -629,6 +757,8 @@ func listen1(ip string, port int, tlsConfig *tls.Config, name string, kinds []st

 // Serve starts serving on the initialized listeners.
 func Serve() {
+	loadStaticGzipCache(mox.DataDirPath("tmp/httpstaticcompresscache"), 512*1024*1024)
+
 	go webadmin.ManageAuthCache()
 	go webaccount.ImportManage()

--- a/http/webserver.go
+++ b/http/webserver.go
@ -11,6 +11,7 @@ import (
 	"fmt"
 	htmltemplate "html/template"
 	"io"
+	"io/fs"
 	golog "log"
 	"net"
 	"net/http"
@ -76,11 +77,14 @@ func WebHandle(w *loggingWriter, r *http.Request, host dns.Domain) (handled bool
 			u.Scheme = "https"
 			u.Host = h.DNSDomain.Name()
 			w.Handler = h.Name
+			w.Compress = h.Compress
 			http.Redirect(w, r, u.String(), http.StatusPermanentRedirect)
 			return true
 		}

-		if h.WebStatic != nil && HandleStatic(h.WebStatic, w, r) {
+		// We don't want the loggingWriter to override the static handler's decisions to compress.
+		w.Compress = h.Compress
+		if h.WebStatic != nil && HandleStatic(h.WebStatic, h.Compress, w, r) {
 			w.Handler = h.Name
 			return true
 		}
@ -93,6 +97,7 @@ func WebHandle(w *loggingWriter, r *http.Request, host dns.Domain) (handled bool
 			return true
 		}
 	}
+	w.Compress = false
 	return false
 }

@ -143,7 +148,7 @@ table > tbody > tr:nth-child(odd) { background-color: #f8f8f8; }
 // slash is written. If a directory is requested and an index.html exists, that
 // file is returned. Otherwise, for directories with ListFiles configured, a
 // directory listing is returned.
-func HandleStatic(h *config.WebStatic, w http.ResponseWriter, r *http.Request) (handled bool) {
+func HandleStatic(h *config.WebStatic, compress bool, w http.ResponseWriter, r *http.Request) (handled bool) {
 	log := func() *mlog.Log {
 		return xlog.WithContext(r.Context())
 	}
@ -174,13 +179,24 @@ func HandleStatic(h *config.WebStatic, w http.ResponseWriter, r *http.Request) (
 	// fspath will not have a trailing slash anymore, we'll correct for it
 	// later when the path turns out to be file instead of a directory.

-	serveFile := func(name string, mtime time.Time, content *os.File) {
+	serveFile := func(name string, fi fs.FileInfo, content *os.File) {
 		// ServeContent only sets a content-type if not already present in the response headers.
 		hdr := w.Header()
 		for k, v := range h.ResponseHeaders {
 			hdr.Add(k, v)
 		}
-		http.ServeContent(w, r, name, mtime, content)
+		// We transparently compress here, but still use ServeContent, because it handles
+		// conditional requests, range requests. It's a bit of a hack, but on first write
+		// to staticgzcacheReplacer where we are compressing, we write the full compressed
+		// file instead, and return an error to ServeContent so it stops. We still have all
+		// the useful behaviour (status code and headers) from ServeContent.
+		xw := w
+		if compress && acceptsGzip(r) && compressibleContent(content) {
+			xw = &staticgzcacheReplacer{w, r, content.Name(), content, fi.ModTime(), fi.Size(), 0, false}
+		} else {
+			w.(*loggingWriter).Compress = false
+		}
+		http.ServeContent(xw, r, name, fi.ModTime(), content)
 	}

 	f, err := os.Open(fspath)
@ -206,7 +222,7 @@ func HandleStatic(h *config.WebStatic, w http.ResponseWriter, r *http.Request) (
 					return true
 				}
 				w.Header().Set("Content-Type", "text/html; charset=utf-8")
-				serveFile("index.html", ifi.ModTime(), index)
+				serveFile("index.html", ifi, index)
 				return true
 			}
 			http.Error(w, "403 - permission denied", http.StatusForbidden)
@ -253,7 +269,7 @@ func HandleStatic(h *config.WebStatic, w http.ResponseWriter, r *http.Request) (
 			ifi, err = index.Stat()
 			if err == nil {
 				w.Header().Set("Content-Type", "text/html; charset=utf-8")
-				serveFile("index.html", ifi.ModTime(), index)
+				serveFile("index.html", ifi, index)
 				return true
 			}
 		}
@ -321,7 +337,7 @@ func HandleStatic(h *config.WebStatic, w http.ResponseWriter, r *http.Request) (
 		return true
 	}

-	serveFile(fspath, fi.ModTime(), f)
+	serveFile(fspath, fi, f)
 	return true
 }

--- a/http/webserver_test.go
+++ b/http/webserver_test.go
@ -31,6 +31,8 @@ func TestWebserver(t *testing.T) {
 	mox.ConfigDynamicPath = filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf")
 	mox.MustLoadConfig(true, false)

+	loadStaticGzipCache(mox.DataDirPath("tmp/httpstaticcompresscache"), 1024*1024)
+
 	srv := &serve{Webserver: true}

 	test := func(method, target string, reqhdrs map[string]string, expCode int, expContent string, expHeaders map[string]string) {
@ -66,10 +68,12 @@ func TestWebserver(t *testing.T) {
 	test("GET", "http://schemeredir.example", nil, http.StatusPermanentRedirect, "", map[string]string{"Location": "https://schemeredir.example/"})
 	test("GET", "https://schemeredir.example", nil, http.StatusNotFound, "", nil)

-	test("GET", "http://mox.example/static/", nil, http.StatusOK, "", map[string]string{"X-Test": "mox"})                              // index.html
-	test("GET", "http://mox.example/static/dir/", nil, http.StatusOK, "", map[string]string{"X-Test": "mox"})                          // listing
-	test("GET", "http://mox.example/static/dir", nil, http.StatusTemporaryRedirect, "", map[string]string{"Location": "/static/dir/"}) // redirect to dir
-	test("GET", "http://mox.example/static/bogus", nil, http.StatusNotFound, "", nil)
+	accgzip := map[string]string{"Accept-Encoding": "gzip"}
+	test("GET", "http://mox.example/static/", accgzip, http.StatusOK, "", map[string]string{"X-Test": "mox", "Content-Encoding": "gzip"})       // index.html
+	test("GET", "http://mox.example/static/dir/hi.txt", accgzip, http.StatusOK, "", map[string]string{"X-Test": "mox", "Content-Encoding": ""}) // too small to compress
+	test("GET", "http://mox.example/static/dir/", accgzip, http.StatusOK, "", map[string]string{"X-Test": "mox", "Content-Encoding": "gzip"})   // listing
+	test("GET", "http://mox.example/static/dir", accgzip, http.StatusTemporaryRedirect, "", map[string]string{"Location": "/static/dir/"})      // redirect to dir
+	test("GET", "http://mox.example/static/bogus", accgzip, http.StatusNotFound, "", map[string]string{"Content-Encoding": ""})

 	test("GET", "http://mox.example/nolist/", nil, http.StatusOK, "", nil)            // index.html
 	test("GET", "http://mox.example/nolist/dir/", nil, http.StatusForbidden, "", nil) // no listing
@ -130,6 +134,26 @@ func TestWebserver(t *testing.T) {

 	test("GET", "http://mox.example/bogus", nil, http.StatusNotFound, "", nil)         // path not registered.
 	test("GET", "http://bogus.mox.example/static/", nil, http.StatusNotFound, "", nil) // domain not registered.
+
+	npaths := len(staticgzcache.paths)
+	if npaths != 1 {
+		t.Fatalf("%d file(s) in staticgzcache, expected 1", npaths)
+	}
+	loadStaticGzipCache(mox.DataDirPath("tmp/httpstaticcompresscache"), 1024*1024)
+	npaths = len(staticgzcache.paths)
+	if npaths != 1 {
+		t.Fatalf("%d file(s) in staticgzcache after loading from disk, expected 1", npaths)
+	}
+	loadStaticGzipCache(mox.DataDirPath("tmp/httpstaticcompresscache"), 0)
+	npaths = len(staticgzcache.paths)
+	if npaths != 0 {
+		t.Fatalf("%d file(s) in staticgzcache after setting max size to 0, expected 0", npaths)
+	}
+	loadStaticGzipCache(mox.DataDirPath("tmp/httpstaticcompresscache"), 0)
+	npaths = len(staticgzcache.paths)
+	if npaths != 0 {
+		t.Fatalf("%d file(s) in staticgzcache after setting max size to 0 and reloading from disk, expected 0", npaths)
+	}
 }

 func TestWebsocket(t *testing.T) {
--- a/testdata/webserver/domains.conf
+++ b/testdata/webserver/domains.conf
@ -21,9 +21,10 @@ WebHandlers:
 		Domain: mox.example
 		PathRegexp: ^/static/
 		DontRedirectPlainHTTP: true
+		Compress: true
 		WebStatic:
 			# This is run from the http package.
-			Root: ../testdata/web
+			Root: ../testdata/webserver
 			ListFiles: true
 			ResponseHeaders:
 				X-Test: mox
@ -35,14 +36,14 @@ WebHandlers:
 		WebStatic:
 			StripPrefix: /nolist/
 			# This is run from the http package.
-			Root: ../testdata/web/static
+			Root: ../testdata/webserver/static
 	-
 		LogName: httpsredir
 		Domain: mox.example
 		PathRegexp: ^/tls/
 		WebStatic:
 			# This is run from the http package.
-			Root: ../testdata/web/static
+			Root: ../testdata/webserver/static
 	-
 		LogName: baseurlonly
 		Domain: mox.example
--- a/testdata/webserver/static/index.html
+++ b/testdata/webserver/static/index.html
@ -1 +1,19 @@
 html
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
+when larger than 1kb, it will be transparently compressed.
--- a/webadmin/admin.html
+++ b/webadmin/admin.html
@ -1938,7 +1938,8 @@ const webserver = async () => {
 						dom.td('LogName', attr({title: 'Name used during logging for requests matching this handler. If empty, the index of the handler in the list is used.'})),
 						dom.td('Domain', attr({title: 'Request must be for this domain to match this handler.'})),
 						dom.td('Path Regexp', attr({title: 'Request must match this path regular expression to match this handler. Must start with with a ^.'})),
-						dom.td('To HTTPS', attr({title: 'Redirect plain HTTP (non-TLS) requests to HTTPS'})),
+						dom.td('To HTTPS', attr({title: 'Redirect plain HTTP (non-TLS) requests to HTTPS.'})),
+						dom.td('Compress', attr({title: 'Transparently compress responses (currently with gzip) if the client supports it, the status is 200 OK, no Content-Encoding is set on the response yet and the Content-Type of the response hints that the data is compressible (text/..., specific application/... and .../...+json and .../...+xml). For static files only, a cache with compressed files is kept.'})),
 					),
 					dom.tr(
 						dom.td(
@ -1953,6 +1954,9 @@ const webserver = async () => {
 						dom.td(
 							row.ToHTTPS=dom.input(attr({type: 'checkbox', title: 'Redirect plain HTTP (non-TLS) requests to HTTPS'}), !wh.DontRedirectPlainHTTP ? attr({checked: ''}) : []),
 						),
+						dom.td(
+							row.Compress=dom.input(attr({type: 'checkbox', title: 'Transparently compress responses.'}), wh.Compress ? attr({checked: ''}) : []),
+						),
 					),
 				),
 				// Replaced with a call to makeType, below (and later when switching types).
@ -2008,6 +2012,7 @@ const webserver = async () => {
 				Domain: row.Domain.value,
 				PathRegexp: row.PathRegexp.value,
 				DontRedirectPlainHTTP: !row.ToHTTPS.checked,
+				Compress: row.Compress.checked,
 			}
 			const s = row.type.value
 			const details = row.getDetails()
--- a/webadmin/adminapi.json
+++ b/webadmin/adminapi.json
@ -3061,6 +3061,13 @@
 						"bool"
 					]
 				},
+				{
+					"Name": "Compress",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
 				{
 					"Name": "WebStatic",
 					"Docs": "",
--- a/webmail/webmail.go
+++ b/webmail/webmail.go
@ -224,6 +224,7 @@ func (m *merged) serve(ctx context.Context, log *mlog.Log, w http.ResponseWriter
 	gz := acceptsGzip(r)
 	var out []byte
 	var mtime time.Time
+	var origSize int64

 	func() {
 		m.Lock()
@ -265,13 +266,14 @@ func (m *merged) serve(ctx context.Context, log *mlog.Log, w http.ResponseWriter
 				xcheckf(ctx, err, "gzipping combined html")
 				m.combinedGzip = b.Bytes()
 			}
+			origSize = int64(len(out))
 			out = m.combinedGzip
 		}
 		mtime = m.mtime
 	}()

 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	http.ServeContent(gzipInjector{w, gz}, r, "", mtime, bytes.NewReader(out))
+	http.ServeContent(gzipInjector{w, gz, origSize}, r, "", mtime, bytes.NewReader(out))
 }

 // gzipInjector is a http.ResponseWriter that optionally injects a
@ -283,6 +285,7 @@ func (m *merged) serve(ctx context.Context, log *mlog.Log, w http.ResponseWriter
 type gzipInjector struct {
 	http.ResponseWriter // Keep most methods.
 	gz                  bool
+	origSize            int64
 }

 // WriteHeader adds a Content-Encoding: gzip header before actually writing the
@ -290,6 +293,9 @@ type gzipInjector struct {
 func (w gzipInjector) WriteHeader(statusCode int) {
 	if w.gz && statusCode == http.StatusOK {
 		w.ResponseWriter.Header().Set("Content-Encoding", "gzip")
+		if lw, ok := w.ResponseWriter.(interface{ SetUncompressedSize(int64) }); ok {
+			lw.SetUncompressedSize(w.origSize)
+		}
 	}
 	w.ResponseWriter.WriteHeader(statusCode)
 }