reverseproxy: add health_upstream subdirective (#6451)

* Add health_upstream

Signed-off-by: Dylan Schultz <9121234+dylanschultzie@users.noreply.github.com>

* Add health_upstream to caddyfile parsing

* Add Active Upstream case for health checks

* Update ignore health port comment

Signed-off-by: Dylan Schultz <9121234+dylanschultzie@users.noreply.github.com>

* Update Upstream json doc

Signed-off-by: Dylan Schultz <9121234+dylanschultzie@users.noreply.github.com>

* Update modules/caddyhttp/reverseproxy/healthchecks.go

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* Use error rather than log for health_port override

Signed-off-by: Dylan Schultz <9121234+dylanschultzie@users.noreply.github.com>

* Add comment about port being ignore if using upstream

Signed-off-by: Dylan Schultz <9121234+dylanschultzie@users.noreply.github.com>

---------

Signed-off-by: Dylan Schultz <9121234+dylanschultzie@users.noreply.github.com>
Co-authored-by: Francis Lavoie <lavofr@gmail.com>

modules/caddyhttp/reverseproxy/caddyfile.go

@@ -16,6 +16,7 @@ package reverseproxy
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"reflect"
 	"strconv"
@@ -354,6 +355,26 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			h.HealthChecks.Active.Path = d.Val()
 			caddy.Log().Named("config.adapter.caddyfile").Warn("the 'health_path' subdirective is deprecated, please use 'health_uri' instead!")
 
+		case "health_upstream":
+			if !d.NextArg() {
+				return d.ArgErr()
+			}
+			if h.HealthChecks == nil {
+				h.HealthChecks = new(HealthChecks)
+			}
+			if h.HealthChecks.Active == nil {
+				h.HealthChecks.Active = new(ActiveHealthChecks)
+			}
+			_, port, err := net.SplitHostPort(d.Val())
+			if err != nil {
+				return d.Errf("health_upstream is malformed '%s': %v", d.Val(), err)
+			}
+			_, err = strconv.Atoi(port)
+			if err != nil {
+				return d.Errf("bad port number '%s': %v", d.Val(), err)
+			}
+			h.HealthChecks.Active.Upstream = d.Val()
+
 		case "health_port":
 			if !d.NextArg() {
 				return d.ArgErr()
@@ -364,6 +385,9 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			if h.HealthChecks.Active == nil {
 				h.HealthChecks.Active = new(ActiveHealthChecks)
 			}
+			if h.HealthChecks.Active.Upstream != "" {
+				return d.Errf("the 'health_port' subdirective is ignored if 'health_upstream' is used!")
+			}
 			portNum, err := strconv.Atoi(d.Val())
 			if err != nil {
 				return d.Errf("bad port number '%s': %v", d.Val(), err)

modules/caddyhttp/reverseproxy/healthchecks.go

@@ -75,8 +75,16 @@ type ActiveHealthChecks struct {
 	// The URI (path and query) to use for health checks
 	URI string `json:"uri,omitempty"`
 
+	// The host:port to use (if different from the upstream's dial address)
+	// for health checks. This should be used in tandem with `health_header` and
+	// `{http.reverse_proxy.active.target_upstream}`. This can be helpful when
+	// creating an intermediate service to do a more thorough health check.
+	// If upstream is set, the active health check port is ignored.
+	Upstream string `json:"upstream,omitempty"`
+
 	// The port to use (if different from the upstream's dial
-	// address) for health checks.
+	// address) for health checks. If active upstream is set,
+	// this value is ignored.
 	Port int `json:"port,omitempty"`
 
 	// HTTP headers to set on health check requests.
@@ -173,9 +181,14 @@ func (a *ActiveHealthChecks) Provision(ctx caddy.Context, h *Handler) error {
 	}
 
 	for _, upstream := range h.Upstreams {
-		// if there's an alternative port for health-check provided in the config,
+		// if there's an alternative upstream for health-check provided in the config,
-		// then use it, otherwise use the port of upstream.
+		// then use it, otherwise use the upstream's dial address. if upstream is used,
-		if a.Port != 0 {
+		// then the port is ignored.
+		if a.Upstream != "" {
+			upstream.activeHealthCheckUpstream = a.Upstream
+		} else if a.Port != 0 {
+			// if there's an alternative port for health-check provided in the config,
+			// then use it, otherwise use the port of upstream.
 			upstream.activeHealthCheckPort = a.Port
 		}
 	}
@@ -350,7 +363,12 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, networ
 	if err != nil {
 		host = hostAddr
 	}
-	if h.HealthChecks.Active.Port != 0 {
+
+	// ignore active health check port if active upstream is provided as the
+	// active upstream already contains the replacement port
+	if h.HealthChecks.Active.Upstream != "" {
+		u.Host = h.HealthChecks.Active.Upstream
+	} else if h.HealthChecks.Active.Port != 0 {
 		port := strconv.Itoa(h.HealthChecks.Active.Port)
 		u.Host = net.JoinHostPort(host, port)
 	}

modules/caddyhttp/reverseproxy/hosts.go

@@ -57,10 +57,11 @@ type Upstream struct {
 	// HeaderAffinity string
 	// IPAffinity     string
 
-	activeHealthCheckPort int
+	activeHealthCheckPort     int
-	healthCheckPolicy     *PassiveHealthChecks
+	activeHealthCheckUpstream string
-	cb                    CircuitBreaker
+	healthCheckPolicy         *PassiveHealthChecks
-	unhealthy             int32 // accessed atomically; status from active health checker
+	cb                        CircuitBreaker
+	unhealthy                 int32 // accessed atomically; status from active health checker
 }
 
 // (pointer receiver necessary to avoid a race condition, since