caddyfile: Populate regexp matcher names by default (#6145)

* caddyfile: Populate regexp matcher names by default

* Some lint cleanup that my VSCode complained about

* Pass down matcher name through expression matcher

* Compat with #6113: fix adapt test, set both styles in replacer

caddyconfig/caddyfile/dispenser.go

@@ -30,6 +30,10 @@ type Dispenser struct {
 	tokens  []Token
 	cursor  int
 	nesting int
+
+	// A map of arbitrary context data that can be used
+	// to pass through some information to unmarshalers.
+	context map[string]any
 }
 
 // NewDispenser returns a Dispenser filled with the given tokens.
@@ -454,6 +458,34 @@ func (d *Dispenser) DeleteN(amount int) []Token {
 	return d.tokens
 }
 
+// SetContext sets a key-value pair in the context map.
+func (d *Dispenser) SetContext(key string, value any) {
+	if d.context == nil {
+		d.context = make(map[string]any)
+	}
+	d.context[key] = value
+}
+
+// GetContext gets the value of a key in the context map.
+func (d *Dispenser) GetContext(key string) any {
+	if d.context == nil {
+		return nil
+	}
+	return d.context[key]
+}
+
+// GetContextString gets the value of a key in the context map
+// as a string, or an empty string if the key does not exist.
+func (d *Dispenser) GetContextString(key string) string {
+	if d.context == nil {
+		return ""
+	}
+	if val, ok := d.context[key].(string); ok {
+		return val
+	}
+	return ""
+}
+
 // isNewLine determines whether the current token is on a different
 // line (higher line number) than the previous token. It handles imported
 // tokens correctly. If there isn't a previous token, it returns true.
@@ -485,3 +517,5 @@ func (d *Dispenser) isNextOnNewLine() bool {
 	next := d.tokens[d.cursor+1]
 	return isNextOnNewLine(curr, next)
 }
+
+const MatcherNameCtxKey = "matcher_name"

caddyconfig/httpcaddyfile/httptype.go

@@ -1397,6 +1397,14 @@ func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.M
 	// given a matcher name and the tokens following it, parse
 	// the tokens as a matcher module and record it
 	makeMatcher := func(matcherName string, tokens []caddyfile.Token) error {
+		// create a new dispenser from the tokens
+		dispenser := caddyfile.NewDispenser(tokens)
+
+		// set the matcher name (without @) in the dispenser context so
+		// that matcher modules can access it to use it as their name
+		// (e.g. regexp matchers which use the name for capture groups)
+		dispenser.SetContext(caddyfile.MatcherNameCtxKey, definitionName[1:])
+
 		mod, err := caddy.GetModule("http.matchers." + matcherName)
 		if err != nil {
 			return fmt.Errorf("getting matcher module '%s': %v", matcherName, err)
@@ -1405,7 +1413,7 @@ func parseMatcherDefinitions(d *caddyfile.Dispenser, matchers map[string]caddy.M
 		if !ok {
 			return fmt.Errorf("matcher module '%s' is not a Caddyfile unmarshaler", matcherName)
 		}
-		err = unm.UnmarshalCaddyfile(caddyfile.NewDispenser(tokens))
+		err = unm.UnmarshalCaddyfile(dispenser)
 		if err != nil {
 			return err
 		}

caddyconfig/httpcaddyfile/serveroptions.go

@@ -291,7 +291,7 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 func applyServerOptions(
 	servers map[string]*caddyhttp.Server,
 	options map[string]any,
-	warnings *[]caddyconfig.Warning,
+	_ *[]caddyconfig.Warning,
 ) error {
 	serverOpts, ok := options["servers"].([]serverOptions)
 	if !ok {

caddyconfig/httpcaddyfile/tlsapp.go

@@ -487,7 +487,11 @@ func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]any) e
 // for any other automation policies. A nil policy (and no error) will be
 // returned if there are no default/global options. However, if always is
 // true, a non-nil value will always be returned (unless there is an error).
-func newBaseAutomationPolicy(options map[string]any, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
+func newBaseAutomationPolicy(
+	options map[string]any,
+	_ []caddyconfig.Warning,
+	always bool,
+) (*caddytls.AutomationPolicy, error) {
 	issuers, hasIssuers := options["cert_issuer"]
 	_, hasLocalCerts := options["local_certs"]
 	keyType, hasKeyType := options["key_type"]

caddytest/integration/caddyfile_adapt/expression_quotes.caddyfiletest

@@ -84,7 +84,10 @@ abort @e
 											],
 											"match": [
 												{
-													"expression": "{http.error.status_code} == 403"
+													"expression": {
+														"expr": "{http.error.status_code} == 403",
+														"name": "d"
+													}
 												}
 											]
 										},
@@ -97,7 +100,10 @@ abort @e
 											],
 											"match": [
 												{
-													"expression": "{http.error.status_code} == 404"
+													"expression": {
+														"expr": "{http.error.status_code} == 404",
+														"name": "e"
+													}
 												}
 											]
 										}

caddytest/integration/caddyfile_adapt/matcher_syntax.caddyfiletest

@@ -146,6 +146,7 @@
 								{
 									"vars_regexp": {
 										"{http.request.uri}": {
+											"name": "matcher6",
 											"pattern": "\\.([a-f0-9]{6})\\.(css|js)$"
 										}
 									}
@@ -161,7 +162,10 @@
 						{
 							"match": [
 								{
-									"expression": "path('/foo*') \u0026\u0026 method('GET')"
+									"expression": {
+										"expr": "path('/foo*') \u0026\u0026 method('GET')",
+										"name": "matcher7"
+									}
 								}
 							],
 							"handle": [

caddytest/integration/caddyfile_adapt/shorthand_parameterized_placeholders.caddyfiletest

@@ -36,6 +36,7 @@ respond @match "{re.1}"
 											"match": [
 												{
 													"path_regexp": {
+														"name": "match",
 														"pattern": "^/foo(.*)$"
 													}
 												}

context.go

@@ -556,3 +556,15 @@ func (ctx Context) Module() Module {
 	}
 	return ctx.ancestry[len(ctx.ancestry)-1]
 }
+
+// WithValue returns a new context with the given key-value pair.
+func (ctx *Context) WithValue(key, value any) Context {
+	return Context{
+		Context:         context.WithValue(ctx.Context, key, value),
+		moduleInstances: ctx.moduleInstances,
+		cfg:             ctx.cfg,
+		ancestry:        ctx.ancestry,
+		cleanupFuncs:    ctx.cleanupFuncs,
+		exitFuncs:       ctx.exitFuncs,
+	}
+}

modules/caddyhttp/celmatcher.go

@@ -62,7 +62,12 @@ type MatchExpression struct {
 	// The CEL expression to evaluate. Any Caddy placeholders
 	// will be expanded and situated into proper CEL function
 	// calls before evaluating.
-	Expr string
+	Expr string `json:"expr,omitempty"`
+
+	// Name is an optional name for this matcher.
+	// This is used to populate the name for regexp
+	// matchers that appear in the expression.
+	Name string `json:"name,omitempty"`
 
 	expandedExpr string
 	prg          cel.Program
@@ -81,12 +86,36 @@ func (MatchExpression) CaddyModule() caddy.ModuleInfo {
 
 // MarshalJSON marshals m's expression.
 func (m MatchExpression) MarshalJSON() ([]byte, error) {
-	return json.Marshal(m.Expr)
+	// if the name is empty, then we can marshal just the expression string
+	if m.Name == "" {
+		return json.Marshal(m.Expr)
+	}
+	// otherwise, we need to marshal the full object, using an
+	// anonymous struct to avoid infinite recursion
+	return json.Marshal(struct {
+		Expr string `json:"expr"`
+		Name string `json:"name"`
+	}{
+		Expr: m.Expr,
+		Name: m.Name,
+	})
 }
 
 // UnmarshalJSON unmarshals m's expression.
 func (m *MatchExpression) UnmarshalJSON(data []byte) error {
-	return json.Unmarshal(data, &m.Expr)
+	// if the data is a string, then it's just the expression
+	if data[0] == '"' {
+		return json.Unmarshal(data, &m.Expr)
+	}
+	// otherwise, it's a full object, so unmarshal it,
+	// using an temp map to avoid infinite recursion
+	var tmpJson map[string]any
+	err := json.Unmarshal(data, &tmpJson)
+	*m = MatchExpression{
+		Expr: tmpJson["expr"].(string),
+		Name: tmpJson["name"].(string),
+	}
+	return err
 }
 
 // Provision sets ups m.
@@ -109,6 +138,11 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error {
 			matcherLibProducers = append(matcherLibProducers, p)
 		}
 	}
+
+	// add the matcher name to the context so that the matcher name
+	// can be used by regexp matchers being provisioned
+	ctx = ctx.WithValue(MatcherNameCtxKey, m.Name)
+
 	// Assemble the compilation and program options from the different library
 	// producers into a single cel.Library implementation.
 	matcherEnvOpts := []cel.EnvOption{}
@@ -197,6 +231,11 @@ func (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	// quoted string; commonly quotes are used in Caddyfile to
 	// define the expression
 	m.Expr = d.Val()
+
+	// use the named matcher's name, to fill regexp
+	// matchers names by default
+	m.Name = d.GetContextString(caddyfile.MatcherNameCtxKey)
+
 	return nil
 }
 
@@ -673,6 +712,8 @@ var httpRequestObjectType = cel.ObjectType("http.Request")
 // The name of the CEL function which accesses Replacer values.
 const placeholderFuncName = "caddyPlaceholder"
 
+const MatcherNameCtxKey = "matcher_name"
+
 // Interface guards
 var (
 	_ caddy.Provisioner     = (*MatchExpression)(nil)

modules/caddyhttp/celmatcher_test.go

@@ -380,7 +380,9 @@ func TestMatchExpressionMatch(t *testing.T) {
 	for _, tst := range matcherTests {
 		tc := tst
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.expression.Provision(caddy.Context{})
+			caddyCtx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+			defer cancel()
+			err := tc.expression.Provision(caddyCtx)
 			if err != nil {
 				if !tc.wantErr {
 					t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tc.wantErr)
@@ -482,7 +484,9 @@ func TestMatchExpressionProvision(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.expression.Provision(caddy.Context{}); (err != nil) != tt.wantErr {
+			ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+			defer cancel()
+			if err := tt.expression.Provision(ctx); (err != nil) != tt.wantErr {
 				t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})

modules/caddyhttp/fileserver/matcher_test.go

@@ -360,7 +360,9 @@ func TestMatchExpressionMatch(t *testing.T) {
 	for _, tst := range expressionTests {
 		tc := tst
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.expression.Provision(caddy.Context{})
+			caddyCtx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+			defer cancel()
+			err := tc.expression.Provision(caddyCtx)
 			if err != nil {
 				if !tc.wantErr {
 					t.Errorf("MatchExpression.Provision() error = %v, wantErr %v", err, tc.wantErr)

modules/caddyhttp/matchers.go

@@ -675,7 +675,10 @@ func (MatchPathRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 		[]*cel.Type{cel.StringType},
 		func(data ref.Val) (RequestMatcher, error) {
 			pattern := data.(types.String)
-			matcher := MatchPathRE{MatchRegexp{Pattern: string(pattern)}}
+			matcher := MatchPathRE{MatchRegexp{
+				Name:    ctx.Value(MatcherNameCtxKey).(string),
+				Pattern: string(pattern),
+			}}
 			err := matcher.Provision(ctx)
 			return matcher, err
 		},
@@ -694,7 +697,14 @@ func (MatchPathRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 				return nil, err
 			}
 			strParams := params.([]string)
-			matcher := MatchPathRE{MatchRegexp{Name: strParams[0], Pattern: strParams[1]}}
+			name := strParams[0]
+			if name == "" {
+				name = ctx.Value(MatcherNameCtxKey).(string)
+			}
+			matcher := MatchPathRE{MatchRegexp{
+				Name:    name,
+				Pattern: strParams[1],
+			}}
 			err = matcher.Provision(ctx)
 			return matcher, err
 		},
@@ -1023,6 +1033,11 @@ func (m *MatchHeaderRE) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			val = second
 		}
 
+		// Default to the named matcher's name, if no regexp name is provided
+		if name == "" {
+			name = d.GetContextString(caddyfile.MatcherNameCtxKey)
+		}
+
 		// If there's already a pattern for this field
 		// then we would end up overwriting the old one
 		if (*m)[field] != nil {
@@ -1099,7 +1114,10 @@ func (MatchHeaderRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 			}
 			strParams := params.([]string)
 			matcher := MatchHeaderRE{}
-			matcher[strParams[0]] = &MatchRegexp{Pattern: strParams[1], Name: ""}
+			matcher[strParams[0]] = &MatchRegexp{
+				Pattern: strParams[1],
+				Name:    ctx.Value(MatcherNameCtxKey).(string),
+			}
 			err = matcher.Provision(ctx)
 			return matcher, err
 		},
@@ -1118,8 +1136,15 @@ func (MatchHeaderRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {
 				return nil, err
 			}
 			strParams := params.([]string)
+			name := strParams[0]
+			if name == "" {
+				name = ctx.Value(MatcherNameCtxKey).(string)
+			}
 			matcher := MatchHeaderRE{}
-			matcher[strParams[1]] = &MatchRegexp{Pattern: strParams[2], Name: strParams[0]}
+			matcher[strParams[1]] = &MatchRegexp{
+				Pattern: strParams[2],
+				Name:    name,
+			}
 			err = matcher.Provision(ctx)
 			return matcher, err
 		},
@@ -1284,7 +1309,6 @@ type MatchRegexp struct {
 	Pattern string `json:"pattern"`
 
 	compiled *regexp.Regexp
-	phPrefix string
 }
 
 // Provision compiles the regular expression.
@@ -1294,10 +1318,6 @@ func (mre *MatchRegexp) Provision(caddy.Context) error {
 		return fmt.Errorf("compiling matcher regexp %s: %v", mre.Pattern, err)
 	}
 	mre.compiled = re
-	mre.phPrefix = regexpPlaceholderPrefix
-	if mre.Name != "" {
-		mre.phPrefix += "." + mre.Name
-	}
 	return nil
 }
 
@@ -1321,16 +1341,25 @@ func (mre *MatchRegexp) Match(input string, repl *caddy.Replacer) bool {
 
 	// save all capture groups, first by index
 	for i, match := range matches {
-		key := mre.phPrefix + "." + strconv.Itoa(i)
-		repl.Set(key, match)
+		keySuffix := "." + strconv.Itoa(i)
+		if mre.Name != "" {
+			repl.Set(regexpPlaceholderPrefix+"."+mre.Name+keySuffix, match)
+		}
+		repl.Set(regexpPlaceholderPrefix+keySuffix, match)
 	}
 
 	// then by name
 	for i, name := range mre.compiled.SubexpNames() {
-		if i != 0 && name != "" {
-			key := mre.phPrefix + "." + name
-			repl.Set(key, matches[i])
+		// skip the first element (the full match), and empty names
+		if i == 0 || name == "" {
+			continue
 		}
+
+		keySuffix := "." + name
+		if mre.Name != "" {
+			repl.Set(regexpPlaceholderPrefix+"."+mre.Name+keySuffix, matches[i])
+		}
+		repl.Set(regexpPlaceholderPrefix+keySuffix, matches[i])
 	}
 
 	return true
@@ -1357,6 +1386,12 @@ func (mre *MatchRegexp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		default:
 			return d.ArgErr()
 		}
+
+		// Default to the named matcher's name, if no regexp name is provided
+		if mre.Name == "" {
+			mre.Name = d.GetContextString(caddyfile.MatcherNameCtxKey)
+		}
+
 		if d.NextBlock(0) {
 			return d.Err("malformed path_regexp matcher: blocks are not supported")
 		}

modules/caddyhttp/routes.go

@@ -311,7 +311,7 @@ func wrapRoute(route Route) Middleware {
 // we need to pull this particular MiddlewareHandler
 // pointer into its own stack frame to preserve it so it
 // won't be overwritten in future loop iterations.
-func wrapMiddleware(ctx caddy.Context, mh MiddlewareHandler, metrics *Metrics) Middleware {
+func wrapMiddleware(_ caddy.Context, mh MiddlewareHandler, metrics *Metrics) Middleware {
 	handlerToUse := mh
 	if metrics != nil {
 		// wrap the middleware with metrics instrumentation

modules/caddyhttp/vars.go

@@ -242,6 +242,11 @@ func (m *MatchVarsRE) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			val = second
 		}
 
+		// Default to the named matcher's name, if no regexp name is provided
+		if name == "" {
+			name = d.GetContextString(caddyfile.MatcherNameCtxKey)
+		}
+
 		(*m)[field] = &MatchRegexp{Pattern: val, Name: name}
 		if d.NextBlock(0) {
 			return d.Err("malformed vars_regexp matcher: blocks are not supported")