encode,staticfiles: Content negotiation, precompressed files (#4045)

* encode: implement prefer setting

* encode: minimum_length configurable via caddyfile

* encode: configurable content-types which to encode

* file_server: support precompressed files

* encode: use ReponseMatcher for conditional encoding of content

* linting error & documentation of encode.PrecompressedOrder

* encode: allow just one response matcher

also change the namespace of the encoders back, I accidently changed to precompressed >.>
default matchers include a *  to match to any charset, that may be appended

* rounding of the PR

* added integration tests for new caddyfile directives
* improved various doc strings (punctuation and typos)
* added json tag for file_server precompress order and encode matcher

* file_server: add vary header, remove accept-ranges when serving precompressed files

* encode: move Suffix implementation to precompressed modules

caddytest/integration/caddyfile_adapt/encode_options.txt

@@ -0,0 +1,66 @@
+:80
+
+encode gzip zstd {
+	minimum_length 256
+	prefer zstd gzip
+	match {
+		status 2xx 4xx 500
+		header Content-Type text/*
+		header Content-Type application/json*
+		header Content-Type application/javascript*
+		header Content-Type application/xhtml+xml*
+		header Content-Type application/atom+xml*
+		header Content-Type application/rss+xml*
+		header Content-Type image/svg+xml*
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"encodings": {
+										"gzip": {},
+										"zstd": {}
+									},
+									"handler": "encode",
+									"match": {
+										"headers": {
+											"Content-Type": [
+												"text/*",
+												"application/json*",
+												"application/javascript*",
+												"application/xhtml+xml*",
+												"application/atom+xml*",
+												"application/rss+xml*",
+												"image/svg+xml*"
+											]
+										},
+										"status_code": [
+											2,
+											4,
+											500
+										]
+									},
+									"minimum_length": 256,
+									"prefer": [
+										"zstd",
+										"gzip"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_precompressed.txt

@@ -0,0 +1,41 @@
+:80
+
+file_server {
+	precompressed zstd br gzip
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									],
+									"precompressed": {
+										"br": {},
+										"gzip": {},
+										"zstd": {}
+									},
+									"precompressed_order": [
+										"zstd",
+										"br",
+										"gzip"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt_test.go

@@ -1,7 +1,10 @@
 package integration
 
 import (
+	jsonMod "encoding/json"
+	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -39,6 +42,10 @@ func TestCaddyfileAdaptToJSON(t *testing.T) {
 		// replace windows newlines in the json with unix newlines
 		json = winNewlines.ReplaceAllString(json, "\n")
 
+		// replace os-specific default path for file_server's hide field
+		replacePath, _ := jsonMod.Marshal(fmt.Sprint(".", string(filepath.Separator), "Caddyfile"))
+		json = strings.ReplaceAll(json, `"./Caddyfile"`, string(replacePath))
+
 		// run the test
 		ok := caddytest.CompareAdapt(t, filename, caddyfile, "caddyfile", json)
 		if !ok {

modules/caddyhttp/encode/brotli/brotli_precompressed.go

@@ -0,0 +1,31 @@
+package caddybrotli
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(BrotliPrecompressed{})
+}
+
+// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding.
+type BrotliPrecompressed struct{}
+
+// CaddyModule returns the Caddy module information.
+func (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.br",
+		New: func() caddy.Module { return new(BrotliPrecompressed) },
+	}
+}
+
+// AcceptEncoding returns the name of the encoding as
+// used in the Accept-Encoding request headers.
+func (BrotliPrecompressed) AcceptEncoding() string { return "br" }
+
+// Suffix returns the filename suffix of precompressed files.
+func (BrotliPrecompressed) Suffix() string { return ".br" }
+
+// Interface guards
+var _ encode.Precompressed = (*BrotliPrecompressed)(nil)

modules/caddyhttp/encode/caddyfile.go

@@ -15,7 +15,9 @@
 package encode
 
 import (
-	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
@@ -42,19 +44,29 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 //     encode [<matcher>] <formats...> {
 //         gzip           [<level>]
 //         zstd
+//         minimum_length <length>
+//         prefer         <formats...>
+//         # response matcher block
+//         match {
+//             status <code...>
+//             header <field> [<value>]
+//         }
+//         # or response matcher single line syntax
+//         match [header <field> [<value>]] | [status <code...>]
 //     }
 //
 // Specifying the formats on the first line will use those formats' defaults.
 func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	responseMatchers := make(map[string]caddyhttp.ResponseMatcher)
 	for d.Next() {
 		for _, arg := range d.RemainingArgs() {
 			mod, err := caddy.GetModule("http.encoders." + arg)
 			if err != nil {
-				return fmt.Errorf("finding encoder module '%s': %v", mod, err)
+				return d.Errf("finding encoder module '%s': %v", mod, err)
 			}
 			encoding, ok := mod.New().(Encoding)
 			if !ok {
-				return fmt.Errorf("module %s is not an HTTP encoding", mod)
+				return d.Errf("module %s is not an HTTP encoding", mod)
 			}
 			if enc.EncodingsRaw == nil {
 				enc.EncodingsRaw = make(caddy.ModuleMap)
@@ -63,6 +75,33 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		}
 
 		for d.NextBlock(0) {
+			switch d.Val() {
+			case "minimum_length":
+				if !d.NextArg() {
+					return d.ArgErr()
+				}
+				minLength, err := strconv.Atoi(d.Val())
+				if err != nil {
+					return err
+				}
+				enc.MinLength = minLength
+			case "prefer":
+				var encs []string
+				for d.NextArg() {
+					encs = append(encs, d.Val())
+				}
+				if len(encs) == 0 {
+					return d.ArgErr()
+				}
+				enc.Prefer = encs
+			case "match":
+				err := enc.parseNamedResponseMatcher(d.NewFromNextSegment(), responseMatchers)
+				if err != nil {
+					return err
+				}
+				matcher := responseMatchers["match"]
+				enc.Matcher = &matcher
+			default:
 				name := d.Val()
 				modID := "http.encoders." + name
 				unm, err := caddyfile.UnmarshalModule(d, modID)
@@ -71,7 +110,7 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				}
 				encoding, ok := unm.(Encoding)
 				if !ok {
-				return fmt.Errorf("module %s is not an HTTP encoding; is %T", modID, unm)
+					return d.Errf("module %s is not an HTTP encoding; is %T", modID, unm)
 				}
 				if enc.EncodingsRaw == nil {
 					enc.EncodingsRaw = make(caddy.ModuleMap)
@@ -79,9 +118,75 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
 			}
 		}
+	}
 
 	return nil
 }
 
+// Parse the tokens of a named response matcher.
+//
+//     match {
+//         header <field> [<value>]
+//         status <code...>
+//     }
+//
+// Or, single line syntax:
+//
+//     match [header <field> [<value>]] | [status <code...>]
+//
+func (enc *Encode) parseNamedResponseMatcher(d *caddyfile.Dispenser, matchers map[string]caddyhttp.ResponseMatcher) error {
+	for d.Next() {
+		definitionName := d.Val()
+
+		if _, ok := matchers[definitionName]; ok {
+			return d.Errf("matcher is defined more than once: %s", definitionName)
+		}
+
+		matcher := caddyhttp.ResponseMatcher{}
+		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
+			switch d.Val() {
+			case "header":
+				if matcher.Headers == nil {
+					matcher.Headers = http.Header{}
+				}
+
+				// reuse the header request matcher's unmarshaler
+				headerMatcher := caddyhttp.MatchHeader(matcher.Headers)
+				err := headerMatcher.UnmarshalCaddyfile(d.NewFromNextSegment())
+				if err != nil {
+					return err
+				}
+
+				matcher.Headers = http.Header(headerMatcher)
+			case "status":
+				if matcher.StatusCode == nil {
+					matcher.StatusCode = []int{}
+				}
+
+				args := d.RemainingArgs()
+				if len(args) == 0 {
+					return d.ArgErr()
+				}
+
+				for _, arg := range args {
+					if len(arg) == 3 && strings.HasSuffix(arg, "xx") {
+						arg = arg[:1]
+					}
+					statusNum, err := strconv.Atoi(arg)
+					if err != nil {
+						return d.Errf("bad status value '%s': %v", arg, err)
+					}
+					matcher.StatusCode = append(matcher.StatusCode, statusNum)
+				}
+			default:
+				return d.Errf("unrecognized response matcher %s", d.Val())
+			}
+		}
+
+		matchers[definitionName] = matcher
+	}
+	return nil
+}
+
 // Interface guard
 var _ caddyfile.Unmarshaler = (*Encode)(nil)

modules/caddyhttp/encode/encode.go

@@ -23,6 +23,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"sort"
 	"strconv"
@@ -43,12 +44,16 @@ type Encode struct {
 	// will be chosen based on the client's Accept-Encoding header.
 	EncodingsRaw caddy.ModuleMap `json:"encodings,omitempty" caddy:"namespace=http.encoders"`
 
-	// If the client has no strong preference, choose this encoding. TODO: Not yet implemented
-	// Prefer    []string `json:"prefer,omitempty"`
+	// If the client has no strong preference, choose these encodings in order.
+	Prefer []string `json:"prefer,omitempty"`
 
 	// Only encode responses that are at least this many bytes long.
 	MinLength int `json:"minimum_length,omitempty"`
 
+	// Only encode responses that match against this ResponseMmatcher.
+	// The default is a collection of text-based Content-Type headers.
+	Matcher *caddyhttp.ResponseMatcher `json:"match,omitempty"`
+
 	writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads...
 }
 
@@ -75,11 +80,46 @@ func (enc *Encode) Provision(ctx caddy.Context) error {
 	if enc.MinLength == 0 {
 		enc.MinLength = defaultMinLength
 	}
+
+	if enc.Matcher == nil {
+		// common text-based content types
+		enc.Matcher = &caddyhttp.ResponseMatcher{
+			Headers: http.Header{
+				"Content-Type": []string{
+					"text/*",
+					"application/json*",
+					"application/javascript*",
+					"application/xhtml+xml*",
+					"application/atom+xml*",
+					"application/rss+xml*",
+					"image/svg+xml*",
+				},
+			},
+		}
+	}
+
+	return nil
+}
+
+// Validate ensures that enc's configuration is valid.
+func (enc *Encode) Validate() error {
+	check := make(map[string]bool)
+	for _, encName := range enc.Prefer {
+		if _, ok := enc.writerPools[encName]; !ok {
+			return fmt.Errorf("encoding %s not enabled", encName)
+		}
+
+		if _, ok := check[encName]; ok {
+			return fmt.Errorf("encoding %s is duplicated in prefer", encName)
+		}
+		check[encName] = true
+	}
+
 	return nil
 }
 
 func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
-	for _, encName := range acceptedEncodings(r) {
+	for _, encName := range AcceptedEncodings(r, enc.Prefer) {
 		if _, ok := enc.writerPools[encName]; !ok {
 			continue // encoding not offered
 		}
@@ -150,6 +190,11 @@ func (rw *responseWriter) WriteHeader(status int) {
 	rw.statusCode = status
 }
 
+// Match determines, if encoding should be done based on the ResponseMatcher.
+func (enc *Encode) Match(rw *responseWriter) bool {
+	return enc.Matcher.Match(rw.statusCode, rw.Header())
+}
+
 // Write writes to the response. If the response qualifies,
 // it is encoded using the encoder, which is initialized
 // if not done so already.
@@ -240,7 +285,10 @@ func (rw *responseWriter) Close() error {
 
 // init should be called before we write a response, if rw.buf has contents.
 func (rw *responseWriter) init() {
-	if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength {
+	if rw.Header().Get("Content-Encoding") == "" &&
+		rw.buf.Len() >= rw.config.MinLength &&
+		rw.config.Match(rw) {
+
 		rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
 		rw.w.Reset(rw.ResponseWriter)
 		rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
@@ -250,12 +298,14 @@ func (rw *responseWriter) init() {
 	rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 }
 
-// acceptedEncodings returns the list of encodings that the
-// client supports, in descending order of preference. If
+// AcceptedEncodings returns the list of encodings that the
+// client supports, in descending order of preference.
+// The client preference via q-factor and the server
+// preference via Prefer setting are taken into account. If
 // the Sec-WebSocket-Key header is present then non-identity
 // encodings are not considered. See
 // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
-func acceptedEncodings(r *http.Request) []string {
+func AcceptedEncodings(r *http.Request, preferredOrder []string) []string {
 	acceptEncHeader := r.Header.Get("Accept-Encoding")
 	websocketKey := r.Header.Get("Sec-WebSocket-Key")
 	if acceptEncHeader == "" {
@@ -292,18 +342,29 @@ func acceptedEncodings(r *http.Request) []string {
 			continue
 		}
 
+		// set server preference
+		prefOrder := -1
+		for i, p := range preferredOrder {
+			if encName == p {
+				prefOrder = len(preferredOrder) - i
+				break
+			}
+		}
+
 		prefs = append(prefs, encodingPreference{
 			encoding:    encName,
 			q:           qFactor,
+			preferOrder: prefOrder,
 		})
 	}
 
-	// sort preferences by descending q-factor
-	sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q })
-
-	// TODO: If no preference, or same pref for all encodings,
-	// and not websocket, use default encoding ordering (enc.Prefer)
-	// for those which are accepted by the client
+	// sort preferences by descending q-factor first, then by preferOrder
+	sort.Slice(prefs, func(i, j int) bool {
+		if math.Abs(prefs[i].q-prefs[j].q) < 0.00001 {
+			return prefs[i].preferOrder > prefs[j].preferOrder
+		}
+		return prefs[i].q > prefs[j].q
+	})
 
 	prefEncNames := make([]string, len(prefs))
 	for i := range prefs {
@@ -317,6 +378,7 @@ func acceptedEncodings(r *http.Request) []string {
 type encodingPreference struct {
 	encoding    string
 	q           float64
+	preferOrder int
 }
 
 // Encoder is a type which can encode a stream of data.
@@ -332,6 +394,13 @@ type Encoding interface {
 	NewEncoder() Encoder
 }
 
+// Precompressed is a type which returns filename suffix of precompressed
+// file and Accept-Encoding header to use when serving this file.
+type Precompressed interface {
+	AcceptEncoding() string
+	Suffix() string
+}
+
 var bufPool = sync.Pool{
 	New: func() interface{} {
 		return new(bytes.Buffer)
@@ -344,6 +413,7 @@ const defaultMinLength = 512
 // Interface guards
 var (
 	_ caddy.Provisioner           = (*Encode)(nil)
+	_ caddy.Validator             = (*Encode)(nil)
 	_ caddyhttp.MiddlewareHandler = (*Encode)(nil)
 	_ caddyhttp.HTTPInterfaces    = (*responseWriter)(nil)
 )

modules/caddyhttp/encode/encode_test.go

@@ -1,6 +1,8 @@
 package encode
 
 import (
+	"net/http"
+	"sync"
 	"testing"
 )
 
@@ -10,3 +12,252 @@ func BenchmarkOpenResponseWriter(b *testing.B) {
 		enc.openResponseWriter("test", nil)
 	}
 }
+
+func TestPreferOrder(t *testing.T) {
+	testCases := []struct {
+		name     string
+		accept   string
+		prefer   []string
+		expected []string
+	}{
+		{
+			name:     "PreferOrder(): 4 accept, 3 prefer",
+			accept:   "deflate, gzip, br, zstd",
+			prefer:   []string{"zstd", "br", "gzip"},
+			expected: []string{"zstd", "br", "gzip", "deflate"},
+		},
+		{
+			name:     "PreferOrder(): 2 accept, 3 prefer",
+			accept:   "deflate, zstd",
+			prefer:   []string{"zstd", "br", "gzip"},
+			expected: []string{"zstd", "deflate"},
+		},
+		{
+			name:     "PreferOrder(): 2 accept (1 empty), 3 prefer",
+			accept:   "gzip,,zstd",
+			prefer:   []string{"zstd", "br", "gzip"},
+			expected: []string{"zstd", "gzip", ""},
+		},
+		{
+			name:     "PreferOrder(): 1 accept, 2 prefer",
+			accept:   "gzip",
+			prefer:   []string{"zstd", "gzip"},
+			expected: []string{"gzip"},
+		},
+		{
+			name:     "PreferOrder(): 4 accept (1 duplicate), 1 prefer",
+			accept:   "deflate, gzip, br, br",
+			prefer:   []string{"br"},
+			expected: []string{"br", "br", "deflate", "gzip"},
+		},
+		{
+			name:     "PreferOrder(): empty accept, 0 prefer",
+			accept:   "",
+			prefer:   []string{},
+			expected: []string{},
+		},
+		{
+			name:     "PreferOrder(): empty accept, 1 prefer",
+			accept:   "",
+			prefer:   []string{"gzip"},
+			expected: []string{},
+		},
+		{
+			name:     "PreferOrder(): with q-factor",
+			accept:   "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd",
+			prefer:   []string{"gzip"},
+			expected: []string{"zstd", "deflate", "gzip", "br"},
+		},
+		{
+			name:     "PreferOrder(): with q-factor, no prefer",
+			accept:   "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd",
+			prefer:   []string{},
+			expected: []string{"zstd", "deflate", "gzip", "br"},
+		},
+		{
+			name:     "PreferOrder(): q-factor=0 filtered out",
+			accept:   "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0",
+			prefer:   []string{"gzip"},
+			expected: []string{"br", "gzip", "deflate"},
+		},
+		{
+			name:     "PreferOrder(): q-factor=0 filtered out, no prefer",
+			accept:   "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0",
+			prefer:   []string{},
+			expected: []string{"br", "gzip", "deflate"},
+		},
+		{
+			name:     "PreferOrder(): with invalid q-factor",
+			accept:   "br, deflate, gzip;q=2, zstd;q=0.1",
+			prefer:   []string{"zstd", "gzip"},
+			expected: []string{"gzip", "br", "deflate", "zstd"},
+		},
+		{
+			name:     "PreferOrder(): with invalid q-factor, no prefer",
+			accept:   "br, deflate, gzip;q=2, zstd;q=0.1",
+			prefer:   []string{},
+			expected: []string{"br", "deflate", "gzip", "zstd"},
+		},
+	}
+
+	enc := new(Encode)
+	r, _ := http.NewRequest("", "", nil)
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+
+			if test.accept == "" {
+				r.Header.Del("Accept-Encoding")
+			} else {
+				r.Header.Set("Accept-Encoding", test.accept)
+			}
+			enc.Prefer = test.prefer
+			result := AcceptedEncodings(r, enc.Prefer)
+			if !sliceEqual(result, test.expected) {
+				t.Errorf("AcceptedEncodings() actual: %s expected: %s",
+					result,
+					test.expected)
+			}
+		})
+	}
+}
+
+func sliceEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func TestValidate(t *testing.T) {
+	type testCase struct {
+		name    string
+		prefer  []string
+		wantErr bool
+	}
+
+	var err error
+	var testCases []testCase
+	enc := new(Encode)
+
+	enc.writerPools = map[string]*sync.Pool{
+		"zstd": nil,
+		"gzip": nil,
+		"br":   nil,
+	}
+	testCases = []testCase{
+		{
+			name:    "ValidatePrefer (zstd, gzip & br enabled): valid order with all encoder",
+			prefer:  []string{"zstd", "br", "gzip"},
+			wantErr: false,
+		},
+		{
+			name:    "ValidatePrefer (zstd, gzip & br enabled): valid order with 2 out of 3 encoders",
+			prefer:  []string{"br", "gzip"},
+			wantErr: false,
+		},
+		{
+			name:    "ValidatePrefer (zstd, gzip & br enabled): valid order with 1 out of 3 encoders",
+			prefer:  []string{"gzip"},
+			wantErr: false,
+		},
+		{
+			name:    "ValidatePrefer (zstd, gzip & br enabled): 1 duplicated (once) encoder",
+			prefer:  []string{"gzip", "zstd", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd, gzip & br enabled): 1 not enabled encoder in prefer list",
+			prefer:  []string{"br", "zstd", "gzip", "deflate"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd, gzip & br enabled): no prefer list",
+			prefer:  []string{},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			enc.Prefer = test.prefer
+			err = enc.Validate()
+			if (err != nil) != test.wantErr {
+				t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
+			}
+		})
+	}
+
+	enc.writerPools = map[string]*sync.Pool{
+		"zstd": nil,
+		"gzip": nil,
+	}
+	testCases = []testCase{
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 1 not enabled encoder in prefer list",
+			prefer:  []string{"zstd", "br", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 2 not enabled encoder in prefer list",
+			prefer:  []string{"br", "zstd", "gzip", "deflate"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): only not enabled encoder in prefer list",
+			prefer:  []string{"deflate", "br", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 1 duplicated (once) encoder in prefer list",
+			prefer:  []string{"gzip", "zstd", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 1 duplicated (twice) encoder in prefer list",
+			prefer:  []string{"gzip", "zstd", "gzip", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 1 duplicated encoder in prefer list",
+			prefer:  []string{"zstd", "zstd", "gzip", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 1 duplicated not enabled encoder in prefer list",
+			prefer:  []string{"br", "br", "gzip"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): 2 duplicated not enabled encoder in prefer list",
+			prefer:  []string{"br", "deflate", "br", "deflate"},
+			wantErr: true,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): valid order zstd first",
+			prefer:  []string{"zstd", "gzip"},
+			wantErr: false,
+		},
+		{
+			name:    "ValidatePrefer (zstd & gzip enabled): valid order gzip first",
+			prefer:  []string{"gzip", "zstd"},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			enc.Prefer = test.prefer
+			err = enc.Validate()
+			if (err != nil) != test.wantErr {
+				t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
+			}
+		})
+
+	}
+}

modules/caddyhttp/encode/gzip/gzip_precompressed.go

@@ -0,0 +1,28 @@
+package caddygzip
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(GzipPrecompressed{})
+}
+
+// GzipPrecompressed provides the file extension for files precompressed with gzip encoding.
+type GzipPrecompressed struct {
+	Gzip
+}
+
+// CaddyModule returns the Caddy module information.
+func (GzipPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.gzip",
+		New: func() caddy.Module { return new(GzipPrecompressed) },
+	}
+}
+
+// Suffix returns the filename suffix of precompressed files.
+func (GzipPrecompressed) Suffix() string { return ".gz" }
+
+var _ encode.Precompressed = (*GzipPrecompressed)(nil)

modules/caddyhttp/encode/zstd/zstd_precompressed.go

@@ -0,0 +1,28 @@
+package caddyzstd
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(ZstdPrecompressed{})
+}
+
+// ZstdPrecompressed provides the file extension for files precompressed with zstandard encoding.
+type ZstdPrecompressed struct {
+	Zstd
+}
+
+// CaddyModule returns the Caddy module information.
+func (ZstdPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.zstd",
+		New: func() caddy.Module { return new(ZstdPrecompressed) },
+	}
+}
+
+// Suffix returns the filename suffix of precompressed files.
+func (ZstdPrecompressed) Suffix() string { return ".zst" }
+
+var _ encode.Precompressed = (*ZstdPrecompressed)(nil)

modules/caddyhttp/fileserver/caddyfile.go

@@ -19,8 +19,10 @@ import (
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
 )
 
@@ -37,6 +39,7 @@ func init() {
 //        hide          <files...>
 //        index         <files...>
 //        browse        [<template_file>]
+//        precompressed <formats...>
 //    }
 //
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
@@ -77,6 +80,26 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				}
 				fsrv.Browse = new(Browse)
 				h.Args(&fsrv.Browse.TemplateFile)
+			case "precompressed":
+				var order []string
+				for h.NextArg() {
+					modID := "http.precompressed." + h.Val()
+					mod, err := caddy.GetModule(modID)
+					if err != nil {
+						return nil, h.Errf("getting module named '%s': %v", modID, err)
+					}
+					inst := mod.New()
+					precompress, ok := inst.(encode.Precompressed)
+					if !ok {
+						return nil, h.Errf("module %s is not a precompressor; is %T", modID, inst)
+					}
+					if fsrv.PrecompressedRaw == nil {
+						fsrv.PrecompressedRaw = make(caddy.ModuleMap)
+					}
+					fsrv.PrecompressedRaw[h.Val()] = caddyconfig.JSON(precompress, nil)
+					order = append(order, h.Val())
+				}
+				fsrv.PrecompressedOrder = order
 			default:
 				return nil, h.Errf("unknown subdirective '%s'", h.Val())
 			}

modules/caddyhttp/fileserver/staticfiles.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
 	"go.uber.org/zap"
 )
 
@@ -79,6 +80,16 @@ type FileServer struct {
 	// a 404 error. By default, this is false (disabled).
 	PassThru bool `json:"pass_thru,omitempty"`
 
+	// Selection of encoders to use to check for precompressed files.
+	PrecompressedRaw caddy.ModuleMap `json:"precompressed,omitempty" caddy:"namespace=http.precompressed"`
+
+	// If the client has no strong preference (q-factor), choose these encodings in order.
+	// If no order specified here, the first encoding from the Accept-Encoding header
+	// that both client and server support is used
+	PrecompressedOrder []string `json:"precompressed_order,omitempty"`
+
+	precompressors map[string]encode.Precompressed
+
 	logger *zap.Logger
 }
 
@@ -129,6 +140,32 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		}
 	}
 
+	mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw")
+	if err != nil {
+		return fmt.Errorf("loading encoder modules: %v", err)
+	}
+	for modName, modIface := range mods.(map[string]interface{}) {
+		p, ok := modIface.(encode.Precompressed)
+		if !ok {
+			return fmt.Errorf("module %s is not precompressor", modName)
+		}
+		ae := p.AcceptEncoding()
+		if ae == "" {
+			return fmt.Errorf("precompressor does not specify an Accept-Encoding value")
+		}
+		suffix := p.Suffix()
+		if suffix == "" {
+			return fmt.Errorf("precompressor does not specify a Suffix value")
+		}
+		if _, ok := fsrv.precompressors[ae]; ok {
+			return fmt.Errorf("precompressor already added: %s", ae)
+		}
+		if fsrv.precompressors == nil {
+			fsrv.precompressors = make(map[string]encode.Precompressed)
+		}
+		fsrv.precompressors[ae] = p
+	}
+
 	return nil
 }
 
@@ -205,8 +242,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		return fsrv.notFound(w, r, next)
 	}
 
-	// TODO: content negotiation (brotli sidecar files, etc...)
-
 	// one last check to ensure the file isn't hidden (we might
 	// have changed the filename from when we last checked)
 	if fileHidden(filename, filesToHide) {
@@ -230,10 +265,42 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		}
 	}
 
+	var file *os.File
+
+	// check for precompressed files
+	for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) {
+		precompress, ok := fsrv.precompressors[ae]
+		if !ok {
+			continue
+		}
+		compressedFilename := filename + precompress.Suffix()
+		compressedInfo, err := os.Stat(compressedFilename)
+		if err != nil || compressedInfo.IsDir() {
+			fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
+			continue
+		}
+		fsrv.logger.Debug("opening compressed sidecar file", zap.String("filename", compressedFilename), zap.Error(err))
+		file, err = fsrv.openFile(compressedFilename, w)
+		if err != nil {
+			fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename), zap.Error(err))
+			if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {
+				return err
+			}
+			continue
+		}
+		defer file.Close()
+		w.Header().Set("Content-Encoding", ae)
+		w.Header().Del("Accept-Ranges")
+		w.Header().Add("Vary", "Accept-Encoding")
+		break
+	}
+
+	// no precompressed file found, use the actual file
+	if file == nil {
 		fsrv.logger.Debug("opening file", zap.String("filename", filename))
 
 		// open the file
-	file, err := fsrv.openFile(filename, w)
+		file, err = fsrv.openFile(filename, w)
 		if err != nil {
 			if herr, ok := err.(caddyhttp.HandlerError); ok &&
 				herr.StatusCode == http.StatusNotFound {
@@ -242,6 +309,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			return err // error is already structured
 		}
 		defer file.Close()
+	}
 
 	// set the ETag - note that a conditional If-None-Match request is handled
 	// by http.ServeContent below, which checks against this ETag value

modules/caddyhttp/standard/imports.go

@@ -5,6 +5,7 @@ import (
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/brotli"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/gzip"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/zstd"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"