diff --git a/modules/caddyhttp/metrics.go b/modules/caddyhttp/metrics.go index 9c0f961e5..111389218 100644 --- a/modules/caddyhttp/metrics.go +++ b/modules/caddyhttp/metrics.go @@ -2,6 +2,7 @@ package caddyhttp import ( "context" + "errors" "net/http" "sync" "time" @@ -137,22 +138,33 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re err := h.mh.ServeHTTP(wrec, r, next) dur := time.Since(start).Seconds() httpMetrics.requestCount.With(labels).Inc() + + observeRequest := func(status int) { + // If the code hasn't been set yet, and we didn't encounter an error, we're + // probably falling through with an empty handler. + if statusLabels["code"] == "" { + // we still sanitize it, even though it's likely to be 0. A 200 is + // returned on fallthrough so we want to reflect that. + statusLabels["code"] = metrics.SanitizeCode(status) + } + + httpMetrics.requestDuration.With(statusLabels).Observe(dur) + httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r))) + httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size())) + } + if err != nil { + var handlerErr HandlerError + if errors.As(err, &handlerErr) { + observeRequest(handlerErr.StatusCode) + } + httpMetrics.requestErrors.With(labels).Inc() + return err } - // If the code hasn't been set yet, and we didn't encounter an error, we're - // probably falling through with an empty handler. - if statusLabels["code"] == "" { - // we still sanitize it, even though it's likely to be 0. A 200 is - // returned on fallthrough so we want to reflect that. - statusLabels["code"] = metrics.SanitizeCode(wrec.Status()) - } - - httpMetrics.requestDuration.With(statusLabels).Observe(dur) - httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r))) - httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size())) + observeRequest(wrec.Status()) return nil } diff --git a/modules/caddyhttp/metrics_test.go b/modules/caddyhttp/metrics_test.go index 6311935ae..8f88549d5 100644 --- a/modules/caddyhttp/metrics_test.go +++ b/modules/caddyhttp/metrics_test.go @@ -5,8 +5,10 @@ import ( "errors" "net/http" "net/http/httptest" + "strings" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -75,6 +77,123 @@ func TestMetricsInstrumentedHandler(t *testing.T) { if actual := w.Result().Header; len(actual) != 0 { t.Errorf("Not empty: expected headers to be empty, but got %#v", actual) } + + // handler returning an error with an HTTP status + mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error { + return Error(http.StatusTooManyRequests, nil) + }) + + ih = newMetricsInstrumentedHandler("foo", mh) + + r = httptest.NewRequest("GET", "/", nil) + w = httptest.NewRecorder() + + if err := ih.ServeHTTP(w, r, nil); err == nil { + t.Errorf("expected error to be propagated") + } + + expected := ` + # HELP caddy_http_request_duration_seconds Histogram of round-trip request durations. + # TYPE caddy_http_request_duration_seconds histogram + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.005"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.01"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.025"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.05"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.1"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.25"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.5"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="2.5"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="5"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="10"} 1 + caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_request_duration_seconds_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1 + # HELP caddy_http_request_size_bytes Total size of the request. Includes body + # TYPE caddy_http_request_size_bytes histogram + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="256"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1024"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4096"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="16384"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="65536"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="262144"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1.048576e+06"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4.194304e+06"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_request_size_bytes_sum{code="200",handler="bar",method="GET",server="UNKNOWN"} 23 + caddy_http_request_size_bytes_count{code="200",handler="bar",method="GET",server="UNKNOWN"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="256"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1024"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4096"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="16384"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="65536"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="262144"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1.048576e+06"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4.194304e+06"} 1 + caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_request_size_bytes_sum{code="200",handler="empty",method="GET",server="UNKNOWN"} 23 + caddy_http_request_size_bytes_count{code="200",handler="empty",method="GET",server="UNKNOWN"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="256"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1024"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4096"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="16384"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="65536"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="262144"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1.048576e+06"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4.194304e+06"} 1 + caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_request_size_bytes_sum{code="429",handler="foo",method="GET",server="UNKNOWN"} 23 + caddy_http_request_size_bytes_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1 + # HELP caddy_http_response_size_bytes Size of the returned response. + # TYPE caddy_http_response_size_bytes histogram + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="256"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1024"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4096"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="16384"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="65536"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="262144"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1.048576e+06"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4.194304e+06"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_response_size_bytes_sum{code="200",handler="bar",method="GET",server="UNKNOWN"} 12 + caddy_http_response_size_bytes_count{code="200",handler="bar",method="GET",server="UNKNOWN"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="256"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1024"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4096"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="16384"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="65536"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="262144"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1.048576e+06"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4.194304e+06"} 1 + caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_response_size_bytes_sum{code="200",handler="empty",method="GET",server="UNKNOWN"} 0 + caddy_http_response_size_bytes_count{code="200",handler="empty",method="GET",server="UNKNOWN"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="256"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1024"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4096"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="16384"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="65536"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="262144"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1.048576e+06"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4.194304e+06"} 1 + caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1 + caddy_http_response_size_bytes_sum{code="429",handler="foo",method="GET",server="UNKNOWN"} 0 + caddy_http_response_size_bytes_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1 + # HELP caddy_http_request_errors_total Number of requests resulting in middleware errors. + # TYPE caddy_http_request_errors_total counter + caddy_http_request_errors_total{handler="bar",server="UNKNOWN"} 1 + caddy_http_request_errors_total{handler="foo",server="UNKNOWN"} 1 + ` + if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), + "caddy_http_request_size_bytes", + "caddy_http_response_size_bytes", + // caddy_http_request_duration_seconds_sum will vary based on how long the test took to run, + // so we check just the _bucket and _count metrics + "caddy_http_request_duration_seconds_bucket", + "caddy_http_request_duration_seconds_count", + "caddy_http_request_errors_total", + ); err != nil { + t.Errorf("received unexpected error: %s", err) + } } type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error