proxyprotocol: Add PROXY protocol support to reverse_proxy, add HTTP listener wrapper (#5424)

Co-authored-by: WeidiDeng <weidi_deng@icloud.com>
Co-authored-by: Francis Lavoie <lavofr@gmail.com>

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/klauspost/compress v1.16.0
 	github.com/klauspost/cpuid/v2 v2.2.4
+	github.com/mastercactapus/proxyprotocol v0.0.4
 	github.com/mholt/acmez v1.1.0
 	github.com/prometheus/client_golang v1.14.0
 	github.com/quic-go/quic-go v0.33.0

go.sum

@@ -443,6 +443,8 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
 github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
+github.com/mastercactapus/proxyprotocol v0.0.4 h1:qSY75IZF30ZqIU9iW1ip3I7gTnm8wRAnGWqPxCBVgq0=
+github.com/mastercactapus/proxyprotocol v0.0.4/go.mod h1:X8FRVEDZz9FkrIoL4QYTBF4Ka4ELwTv0sah0/5NxCPw=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=

modules/caddyhttp/proxyprotocol/listenerwrapper.go

@@ -0,0 +1,68 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxyprotocol
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/mastercactapus/proxyprotocol"
+)
+
+// ListenerWrapper provides PROXY protocol support to Caddy by implementing
+// the caddy.ListenerWrapper interface. It must be loaded before the `tls` listener.
+//
+// Credit goes to https://github.com/mastercactapus/caddy2-proxyprotocol for having
+// initially implemented this as a plugin.
+type ListenerWrapper struct {
+	// Timeout specifies an optional maximum time for
+	// the PROXY header to be received.
+	// If zero, timeout is disabled. Default is 5s.
+	Timeout caddy.Duration `json:"timeout,omitempty"`
+
+	// Allow is an optional list of CIDR ranges to
+	// allow/require PROXY headers from.
+	Allow []string `json:"allow,omitempty"`
+
+	rules []proxyprotocol.Rule
+}
+
+// Provision sets up the listener wrapper.
+func (pp *ListenerWrapper) Provision(ctx caddy.Context) error {
+	rules := make([]proxyprotocol.Rule, 0, len(pp.Allow))
+	for _, s := range pp.Allow {
+		_, n, err := net.ParseCIDR(s)
+		if err != nil {
+			return fmt.Errorf("invalid subnet '%s': %w", s, err)
+		}
+		rules = append(rules, proxyprotocol.Rule{
+			Timeout: time.Duration(pp.Timeout),
+			Subnet:  n,
+		})
+	}
+
+	pp.rules = rules
+
+	return nil
+}
+
+// WrapListener adds PROXY protocol support to the listener.
+func (pp *ListenerWrapper) WrapListener(l net.Listener) net.Listener {
+	pl := proxyprotocol.NewListener(l, time.Duration(pp.Timeout))
+	pl.SetFilter(pp.rules)
+	return pl
+}

modules/caddyhttp/proxyprotocol/module.go

@@ -0,0 +1,75 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxyprotocol
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+func init() {
+	caddy.RegisterModule(ListenerWrapper{})
+}
+
+func (ListenerWrapper) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "caddy.listeners.proxy_protocol",
+		New: func() caddy.Module { return new(ListenerWrapper) },
+	}
+}
+
+// UnmarshalCaddyfile sets up the listener Listenerwrapper from Caddyfile tokens. Syntax:
+//
+//	proxy_protocol {
+//		timeout <duration>
+//		allow <IPs...>
+//	}
+func (w *ListenerWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		// No same-line options are supported
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+
+		for d.NextBlock(0) {
+			switch d.Val() {
+			case "timeout":
+				if !d.NextArg() {
+					return d.ArgErr()
+				}
+				dur, err := caddy.ParseDuration(d.Val())
+				if err != nil {
+					return d.Errf("parsing proxy_protocol timeout duration: %v", err)
+				}
+				w.Timeout = caddy.Duration(dur)
+
+			case "allow":
+				w.Allow = append(w.Allow, d.RemainingArgs()...)
+
+			default:
+				return d.ArgErr()
+			}
+		}
+	}
+	return nil
+}
+
+// Interface guards
+var (
+	_ caddy.Provisioner     = (*ListenerWrapper)(nil)
+	_ caddy.Module          = (*ListenerWrapper)(nil)
+	_ caddy.ListenerWrapper = (*ListenerWrapper)(nil)
+	_ caddyfile.Unmarshaler = (*ListenerWrapper)(nil)
+)

modules/caddyhttp/reverseproxy/caddyfile.go

@@ -918,6 +918,17 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				}
 				h.MaxResponseHeaderSize = int64(size)
 
+			case "proxy_protocol":
+				if !d.NextArg() {
+					return d.ArgErr()
+				}
+				switch proxyProtocol := d.Val(); proxyProtocol {
+				case "v1", "v2":
+					h.ProxyProtocol = proxyProtocol
+				default:
+					return d.Errf("invalid proxy protocol version '%s'", proxyProtocol)
+				}
+
 			case "dial_timeout":
 				if !d.NextArg() {
 					return d.ArgErr()

modules/caddyhttp/reverseproxy/hosts.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/netip"
 	"strconv"
 	"sync/atomic"
 
@@ -259,3 +260,13 @@ var hosts = caddy.NewUsagePool()
 // dialInfoVarKey is the key used for the variable that holds
 // the dial info for the upstream connection.
 const dialInfoVarKey = "reverse_proxy.dial_info"
+
+// proxyProtocolInfoVarKey is the key used for the variable that holds
+// the proxy protocol info for the upstream connection.
+const proxyProtocolInfoVarKey = "reverse_proxy.proxy_protocol_info"
+
+// ProxyProtocolInfo contains information needed to write proxy protocol to a
+// connection to an upstream host.
+type ProxyProtocolInfo struct {
+	AddrPort netip.AddrPort
+}

modules/caddyhttp/reverseproxy/httptransport.go

@@ -29,7 +29,9 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/mastercactapus/proxyprotocol"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
 )
@@ -64,6 +66,10 @@ type HTTPTransport struct {
 	// Maximum number of connections per host. Default: 0 (no limit)
 	MaxConnsPerHost int `json:"max_conns_per_host,omitempty"`
 
+	// If non-empty, which PROXY protocol version to send when
+	// connecting to an upstream. Default: off.
+	ProxyProtocol string `json:"proxy_protocol,omitempty"`
+
 	// How long to wait before timing out trying to connect to
 	// an upstream. Default: `3s`.
 	DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`
@@ -195,6 +201,57 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e
 			return nil, DialError{err}
 		}
 
+		if h.ProxyProtocol != "" {
+			proxyProtocolInfo, ok := caddyhttp.GetVar(ctx, proxyProtocolInfoVarKey).(ProxyProtocolInfo)
+			if !ok {
+				return nil, fmt.Errorf("failed to get proxy protocol info from context")
+			}
+
+			// The src and dst have to be of the some address family. As we don't know the original
+			// dst address (it's kind of impossible to know) and this address is generelly of very
+			// little interest, we just set it to all zeros.
+			var destIP net.IP
+			switch {
+			case proxyProtocolInfo.AddrPort.Addr().Is4():
+				destIP = net.IPv4zero
+			case proxyProtocolInfo.AddrPort.Addr().Is6():
+				destIP = net.IPv6zero
+			default:
+				return nil, fmt.Errorf("unexpected remote addr type in proxy protocol info")
+			}
+
+			// TODO: We should probably migrate away from net.IP to use netip.Addr,
+			// but due to the upstream dependency, we can't do that yet.
+			switch h.ProxyProtocol {
+			case "v1":
+				header := proxyprotocol.HeaderV1{
+					SrcIP:    net.IP(proxyProtocolInfo.AddrPort.Addr().AsSlice()),
+					SrcPort:  int(proxyProtocolInfo.AddrPort.Port()),
+					DestIP:   destIP,
+					DestPort: 0,
+				}
+				caddyCtx.Logger().Debug("sending proxy protocol header v1", zap.Any("header", header))
+				_, err = header.WriteTo(conn)
+			case "v2":
+				header := proxyprotocol.HeaderV2{
+					Command: proxyprotocol.CmdProxy,
+					Src:     &net.TCPAddr{IP: net.IP(proxyProtocolInfo.AddrPort.Addr().AsSlice()), Port: int(proxyProtocolInfo.AddrPort.Port())},
+					Dest:    &net.TCPAddr{IP: destIP, Port: 0},
+				}
+				caddyCtx.Logger().Debug("sending proxy protocol header v2", zap.Any("header", header))
+				_, err = header.WriteTo(conn)
+			default:
+				return nil, fmt.Errorf("unexpected proxy protocol version")
+			}
+
+			if err != nil {
+				// identify this error as one that occurred during
+				// dialing, which can be important when trying to
+				// decide whether to retry a request
+				return nil, DialError{err}
+			}
+		}
+
 		// if read/write timeouts are configured and this is a TCP connection,
 		// enforce the timeouts by wrapping the connection with our own type
 		if tcpConn, ok := conn.(*net.TCPConn); ok && (h.ReadTimeout > 0 || h.WriteTimeout > 0) {
@@ -239,6 +296,14 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e
 		rt.IdleConnTimeout = time.Duration(h.KeepAlive.IdleConnTimeout)
 	}
 
+	// The proxy protocol header can only be sent once right after opening the connection.
+	// So single connection must not be used for multiple requests, which can potentially
+	// come from different clients.
+	if !rt.DisableKeepAlives && h.ProxyProtocol != "" {
+		caddyCtx.Logger().Warn("disabling keepalives, they are incompatible with using PROXY protocol")
+		rt.DisableKeepAlives = true
+	}
+
 	if h.Compression != nil {
 		rt.DisableCompression = !*h.Compression
 	}

modules/caddyhttp/reverseproxy/reverseproxy.go

@@ -688,8 +688,18 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http.
 		req.Header.Set("Upgrade", reqUpType)
 	}
 
+	// Set up the PROXY protocol info
+	address := caddyhttp.GetVar(req.Context(), caddyhttp.ClientIPVarKey).(string)
+	addrPort, err := netip.ParseAddrPort(address)
+	if err != nil {
+		// OK; probably didn't have a port
+		addrPort, _ = netip.ParseAddrPort(address + ":0")
+	}
+	proxyProtocolInfo := ProxyProtocolInfo{AddrPort: addrPort}
+	caddyhttp.SetVar(req.Context(), proxyProtocolInfoVarKey, proxyProtocolInfo)
+
 	// Add the supported X-Forwarded-* headers
-	err := h.addForwardedHeaders(req)
+	err = h.addForwardedHeaders(req)
 	if err != nil {
 		return nil, err
 	}

modules/caddyhttp/standard/imports.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/map"
+	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/proxyprotocol"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/push"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/requestbody"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"