caddyhttp: Implement named routes, invoke directive (#5107)

* caddyhttp: Implement named routes, `invoke` directive

* gofmt

* Add experimental marker

* Adjust route compile comments

caddyconfig/caddyfile/parse.go

@@ -148,7 +148,6 @@ func (p *parser) begin() error {
 	}
 
 	err := p.addresses()
-
 	if err != nil {
 		return err
 	}
@@ -159,6 +158,25 @@ func (p *parser) begin() error {
 		return nil
 	}
 
+	if ok, name := p.isNamedRoute(); ok {
+		// named routes only have one key, the route name
+		p.block.Keys = []string{name}
+		p.block.IsNamedRoute = true
+
+		// we just need a dummy leading token to ease parsing later
+		nameToken := p.Token()
+		nameToken.Text = name
+
+		// get all the tokens from the block, including the braces
+		tokens, err := p.blockTokens(true)
+		if err != nil {
+			return err
+		}
+		tokens = append([]Token{nameToken}, tokens...)
+		p.block.Segments = []Segment{tokens}
+		return nil
+	}
+
 	if ok, name := p.isSnippet(); ok {
 		if p.definedSnippets == nil {
 			p.definedSnippets = map[string][]Token{}
@@ -167,7 +185,7 @@ func (p *parser) begin() error {
 			return p.Errf("redeclaration of previously declared snippet %s", name)
 		}
 		// consume all tokens til matched close brace
-		tokens, err := p.snippetTokens()
+		tokens, err := p.blockTokens(false)
 		if err != nil {
 			return err
 		}
@@ -576,6 +594,15 @@ func (p *parser) closeCurlyBrace() error {
 	return nil
 }
 
+func (p *parser) isNamedRoute() (bool, string) {
+	keys := p.block.Keys
+	// A named route block is a single key with parens, prefixed with &.
+	if len(keys) == 1 && strings.HasPrefix(keys[0], "&(") && strings.HasSuffix(keys[0], ")") {
+		return true, strings.TrimSuffix(keys[0][2:], ")")
+	}
+	return false, ""
+}
+
 func (p *parser) isSnippet() (bool, string) {
 	keys := p.block.Keys
 	// A snippet block is a single key with parens. Nothing else qualifies.
@@ -586,18 +613,24 @@ func (p *parser) isSnippet() (bool, string) {
 }
 
 // read and store everything in a block for later replay.
-func (p *parser) snippetTokens() ([]Token, error) {
+func (p *parser) blockTokens(retainCurlies bool) ([]Token, error) {
-	// snippet must have curlies.
+	// block must have curlies.
 	err := p.openCurlyBrace()
 	if err != nil {
 		return nil, err
 	}
-	nesting := 1 // count our own nesting in snippets
+	nesting := 1 // count our own nesting
 	tokens := []Token{}
+	if retainCurlies {
+		tokens = append(tokens, p.Token())
+	}
 	for p.Next() {
 		if p.Val() == "}" {
 			nesting--
 			if nesting == 0 {
+				if retainCurlies {
+					tokens = append(tokens, p.Token())
+				}
 				break
 			}
 		}
@@ -620,6 +653,7 @@ type ServerBlock struct {
 	HasBraces    bool
 	Keys         []string
 	Segments     []Segment
+	IsNamedRoute bool
 }
 
 // DispenseDirective returns a dispenser that contains

caddyconfig/httpcaddyfile/builtins.go

@@ -48,6 +48,7 @@ func init() {
 	RegisterHandlerDirective("route", parseRoute)
 	RegisterHandlerDirective("handle", parseHandle)
 	RegisterDirective("handle_errors", parseHandleErrors)
+	RegisterHandlerDirective("invoke", parseInvoke)
 	RegisterDirective("log", parseLog)
 	RegisterHandlerDirective("skip_log", parseSkipLog)
 }
@@ -764,6 +765,27 @@ func parseHandleErrors(h Helper) ([]ConfigValue, error) {
 	}, nil
 }
 
+// parseInvoke parses the invoke directive.
+func parseInvoke(h Helper) (caddyhttp.MiddlewareHandler, error) {
+	h.Next() // consume directive
+	if !h.NextArg() {
+		return nil, h.ArgErr()
+	}
+	for h.Next() || h.NextBlock(0) {
+		return nil, h.ArgErr()
+	}
+
+	// remember that we're invoking this name
+	// to populate the server with these named routes
+	if h.State[namedRouteKey] == nil {
+		h.State[namedRouteKey] = map[string]struct{}{}
+	}
+	h.State[namedRouteKey].(map[string]struct{})[h.Val()] = struct{}{}
+
+	// return the handler
+	return &caddyhttp.Invoke{Name: h.Val()}, nil
+}
+
 // parseLog parses the log directive. Syntax:
 //
 //	log {

caddyconfig/httpcaddyfile/directives.go

@@ -65,6 +65,7 @@ var directiveOrder = []string{
 	"templates",
 
 	// special routing & dispatching directives
+	"invoke",
 	"handle",
 	"handle_path",
 	"route",

caddyconfig/httpcaddyfile/httptype.go

@@ -52,8 +52,10 @@ type ServerType struct {
 }
 
 // Setup makes a config from the tokens.
-func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
+func (st ServerType) Setup(
-	options map[string]any) (*caddy.Config, []caddyconfig.Warning, error) {
+	inputServerBlocks []caddyfile.ServerBlock,
+	options map[string]any,
+) (*caddy.Config, []caddyconfig.Warning, error) {
 	var warnings []caddyconfig.Warning
 	gc := counter{new(int)}
 	state := make(map[string]any)
@@ -79,6 +81,11 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 		return nil, warnings, err
 	}
 
+	originalServerBlocks, err = st.extractNamedRoutes(originalServerBlocks, options, &warnings)
+	if err != nil {
+		return nil, warnings, err
+	}
+
 	// replace shorthand placeholders (which are convenient
 	// when writing a Caddyfile) with their actual placeholder
 	// identifiers or variable names
@@ -172,6 +179,18 @@ func (st ServerType) Setup(inputServerBlocks []caddyfile.ServerBlock,
 				result.directive = dir
 				sb.pile[result.Class] = append(sb.pile[result.Class], result)
 			}
+
+			// specially handle named routes that were pulled out from
+			// the invoke directive, which could be nested anywhere within
+			// some subroutes in this directive; we add them to the pile
+			// for this server block
+			if state[namedRouteKey] != nil {
+				for name := range state[namedRouteKey].(map[string]struct{}) {
+					result := ConfigValue{Class: namedRouteKey, Value: name}
+					sb.pile[result.Class] = append(sb.pile[result.Class], result)
+				}
+				state[namedRouteKey] = nil
+			}
 		}
 	}
 
@@ -403,6 +422,77 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 	return serverBlocks[1:], nil
 }
 
+// extractNamedRoutes pulls out any named route server blocks
+// so they don't get parsed as sites, and stores them in options
+// for later.
+func (ServerType) extractNamedRoutes(
+	serverBlocks []serverBlock,
+	options map[string]any,
+	warnings *[]caddyconfig.Warning,
+) ([]serverBlock, error) {
+	namedRoutes := map[string]*caddyhttp.Route{}
+
+	gc := counter{new(int)}
+	state := make(map[string]any)
+
+	// copy the server blocks so we can
+	// splice out the named route ones
+	filtered := append([]serverBlock{}, serverBlocks...)
+	index := -1
+
+	for _, sb := range serverBlocks {
+		index++
+		if !sb.block.IsNamedRoute {
+			continue
+		}
+
+		// splice out this block, because we know it's not a real server
+		filtered = append(filtered[:index], filtered[index+1:]...)
+		index--
+
+		if len(sb.block.Segments) == 0 {
+			continue
+		}
+
+		// zip up all the segments since ParseSegmentAsSubroute
+		// was designed to take a directive+
+		wholeSegment := caddyfile.Segment{}
+		for _, segment := range sb.block.Segments {
+			wholeSegment = append(wholeSegment, segment...)
+		}
+
+		h := Helper{
+			Dispenser:    caddyfile.NewDispenser(wholeSegment),
+			options:      options,
+			warnings:     warnings,
+			matcherDefs:  nil,
+			parentBlock:  sb.block,
+			groupCounter: gc,
+			State:        state,
+		}
+
+		handler, err := ParseSegmentAsSubroute(h)
+		if err != nil {
+			return nil, err
+		}
+		subroute := handler.(*caddyhttp.Subroute)
+		route := caddyhttp.Route{}
+
+		if len(subroute.Routes) == 1 && len(subroute.Routes[0].MatcherSetsRaw) == 0 {
+			// if there's only one route with no matcher, then we can simplify
+			route.HandlersRaw = append(route.HandlersRaw, subroute.Routes[0].HandlersRaw[0])
+		} else {
+			// otherwise we need the whole subroute
+			route.HandlersRaw = []json.RawMessage{caddyconfig.JSONModuleObject(handler, "handler", subroute.CaddyModule().ID.Name(), h.warnings)}
+		}
+
+		namedRoutes[sb.block.Keys[0]] = &route
+	}
+	options["named_routes"] = namedRoutes
+
+	return filtered, nil
+}
+
 // serversFromPairings creates the servers for each pairing of addresses
 // to server blocks. Each pairing is essentially a server definition.
 func (st *ServerType) serversFromPairings(
@@ -542,6 +632,24 @@ func (st *ServerType) serversFromPairings(
 			}
 		}
 
+		// add named routes to the server if 'invoke' was used inside of it
+		configuredNamedRoutes := options["named_routes"].(map[string]*caddyhttp.Route)
+		for _, sblock := range p.serverBlocks {
+			if len(sblock.pile[namedRouteKey]) == 0 {
+				continue
+			}
+			for _, value := range sblock.pile[namedRouteKey] {
+				if srv.NamedRoutes == nil {
+					srv.NamedRoutes = map[string]*caddyhttp.Route{}
+				}
+				name := value.Value.(string)
+				if configuredNamedRoutes[name] == nil {
+					return nil, fmt.Errorf("cannot invoke named route '%s', which was not defined", name)
+				}
+				srv.NamedRoutes[name] = configuredNamedRoutes[name]
+			}
+		}
+
 		// create a subroute for each site in the server block
 		for _, sblock := range p.serverBlocks {
 			matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock)
@@ -1469,6 +1577,7 @@ type sbAddrAssociation struct {
 }
 
 const matcherPrefix = "@"
+const namedRouteKey = "named_route"
 
 // Interface guard
 var _ caddyfile.ServerType = (*ServerType)(nil)

caddytest/integration/caddyfile_adapt/invoke_named_routes.txt

@@ -0,0 +1,154 @@
+&(first) {
+	@first path /first
+	vars @first first 1
+	respond "first"
+}
+
+&(second) {
+	respond "second"
+}
+
+:8881 {
+	invoke first
+	route {
+		invoke second
+	}
+}
+
+:8882 {
+	handle {
+		invoke second
+	}
+}
+
+:8883 {
+	respond "no invoke"
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8881"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "invoke",
+									"name": "first"
+								},
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "invoke",
+													"name": "second"
+												}
+											]
+										}
+									]
+								}
+							]
+						}
+					],
+					"named_routes": {
+						"first": {
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"first": 1,
+													"handler": "vars"
+												}
+											],
+											"match": [
+												{
+													"path": [
+														"/first"
+													]
+												}
+											]
+										},
+										{
+											"handle": [
+												{
+													"body": "first",
+													"handler": "static_response"
+												}
+											]
+										}
+									]
+								}
+							]
+						},
+						"second": {
+							"handle": [
+								{
+									"body": "second",
+									"handler": "static_response"
+								}
+							]
+						}
+					}
+				},
+				"srv1": {
+					"listen": [
+						":8882"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "invoke",
+													"name": "second"
+												}
+											]
+										}
+									]
+								}
+							]
+						}
+					],
+					"named_routes": {
+						"second": {
+							"handle": [
+								{
+									"body": "second",
+									"handler": "static_response"
+								}
+							]
+						}
+					}
+				},
+				"srv2": {
+					"listen": [
+						":8883"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"body": "no invoke",
+									"handler": "static_response"
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

modules/caddyhttp/app.go

@@ -293,11 +293,19 @@ func (app *App) Provision(ctx caddy.Context) error {
 		if srv.Errors != nil {
 			err := srv.Errors.Routes.Provision(ctx)
 			if err != nil {
-				return fmt.Errorf("server %s: setting up server error handling routes: %v", srvName, err)
+				return fmt.Errorf("server %s: setting up error handling routes: %v", srvName, err)
 			}
 			srv.errorHandlerChain = srv.Errors.Routes.Compile(errorEmptyHandler)
 		}
 
+		// provision the named routes (they get compiled at runtime)
+		for name, route := range srv.NamedRoutes {
+			err := route.Provision(ctx, srv.Metrics)
+			if err != nil {
+				return fmt.Errorf("server %s: setting up named route '%s' handlers: %v", name, srvName, err)
+			}
+		}
+
 		// prepare the TLS connection policies
 		err = srv.TLSConnPolicies.Provision(ctx)
 		if err != nil {

modules/caddyhttp/invoke.go

@@ -0,0 +1,56 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyhttp
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/caddyserver/caddy/v2"
+)
+
+func init() {
+	caddy.RegisterModule(Invoke{})
+}
+
+// Invoke implements a handler that compiles and executes a
+// named route that was defined on the server.
+//
+// EXPERIMENTAL: Subject to change or removal.
+type Invoke struct {
+	// Name is the key of the named route to execute
+	Name string `json:"name,omitempty"`
+}
+
+// CaddyModule returns the Caddy module information.
+func (Invoke) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.handlers.invoke",
+		New: func() caddy.Module { return new(Invoke) },
+	}
+}
+
+func (invoke *Invoke) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {
+	server := r.Context().Value(ServerCtxKey).(*Server)
+	if route, ok := server.NamedRoutes[invoke.Name]; ok {
+		return route.Compile(next).ServeHTTP(w, r)
+	}
+	return fmt.Errorf("invoke: route '%s' not found", invoke.Name)
+}
+
+// Interface guards
+var (
+	_ MiddlewareHandler = (*Invoke)(nil)
+)

modules/caddyhttp/routes.go

@@ -120,6 +120,59 @@ func (r Route) String() string {
 		r.Group, r.MatcherSetsRaw, handlersRaw, r.Terminal)
 }
 
+// Provision sets up both the matchers and handlers in the route.
+func (r *Route) Provision(ctx caddy.Context, metrics *Metrics) error {
+	err := r.ProvisionMatchers(ctx)
+	if err != nil {
+		return err
+	}
+	return r.ProvisionHandlers(ctx, metrics)
+}
+
+// ProvisionMatchers sets up all the matchers by loading the
+// matcher modules. Only call this method directly if you need
+// to set up matchers and handlers separately without having
+// to provision a second time; otherwise use Provision instead.
+func (r *Route) ProvisionMatchers(ctx caddy.Context) error {
+	// matchers
+	matchersIface, err := ctx.LoadModule(r, "MatcherSetsRaw")
+	if err != nil {
+		return fmt.Errorf("loading matcher modules: %v", err)
+	}
+	err = r.MatcherSets.FromInterface(matchersIface)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ProvisionHandlers sets up all the handlers by loading the
+// handler modules. Only call this method directly if you need
+// to set up matchers and handlers separately without having
+// to provision a second time; otherwise use Provision instead.
+func (r *Route) ProvisionHandlers(ctx caddy.Context, metrics *Metrics) error {
+	handlersIface, err := ctx.LoadModule(r, "HandlersRaw")
+	if err != nil {
+		return fmt.Errorf("loading handler modules: %v", err)
+	}
+	for _, handler := range handlersIface.([]any) {
+		r.Handlers = append(r.Handlers, handler.(MiddlewareHandler))
+	}
+
+	// pre-compile the middleware handler chain
+	for _, midhandler := range r.Handlers {
+		r.middleware = append(r.middleware, wrapMiddleware(ctx, midhandler, metrics))
+	}
+	return nil
+}
+
+// Compile prepares a middleware chain from the route list.
+// This should only be done once during the request, just
+// before the middleware chain is executed.
+func (r Route) Compile(next Handler) Handler {
+	return wrapRoute(r)(next)
+}
+
 // RouteList is a list of server routes that can
 // create a middleware chain.
 type RouteList []Route
@@ -139,12 +192,7 @@ func (routes RouteList) Provision(ctx caddy.Context) error {
 // to provision a second time; otherwise use Provision instead.
 func (routes RouteList) ProvisionMatchers(ctx caddy.Context) error {
 	for i := range routes {
-		// matchers
+		err := routes[i].ProvisionMatchers(ctx)
-		matchersIface, err := ctx.LoadModule(&routes[i], "MatcherSetsRaw")
-		if err != nil {
-			return fmt.Errorf("route %d: loading matcher modules: %v", i, err)
-		}
-		err = routes[i].MatcherSets.FromInterface(matchersIface)
 		if err != nil {
 			return fmt.Errorf("route %d: %v", i, err)
 		}
@@ -158,25 +206,18 @@ func (routes RouteList) ProvisionMatchers(ctx caddy.Context) error {
 // to provision a second time; otherwise use Provision instead.
 func (routes RouteList) ProvisionHandlers(ctx caddy.Context, metrics *Metrics) error {
 	for i := range routes {
-		handlersIface, err := ctx.LoadModule(&routes[i], "HandlersRaw")
+		err := routes[i].ProvisionHandlers(ctx, metrics)
 		if err != nil {
-			return fmt.Errorf("route %d: loading handler modules: %v", i, err)
+			return fmt.Errorf("route %d: %v", i, err)
-		}
-		for _, handler := range handlersIface.([]any) {
-			routes[i].Handlers = append(routes[i].Handlers, handler.(MiddlewareHandler))
-		}
-
-		// pre-compile the middleware handler chain
-		for _, midhandler := range routes[i].Handlers {
-			routes[i].middleware = append(routes[i].middleware, wrapMiddleware(ctx, midhandler, metrics))
 		}
 	}
 	return nil
 }
 
 // Compile prepares a middleware chain from the route list.
-// This should only be done once: after all the routes have
+// This should only be done either once during provisioning
-// been provisioned, and before serving requests.
+// for top-level routes, or on each request just before the
+// middleware chain is executed for subroutes.
 func (routes RouteList) Compile(next Handler) Handler {
 	mid := make([]Middleware, 0, len(routes))
 	for _, route := range routes {

modules/caddyhttp/server.go

@@ -102,6 +102,16 @@ type Server struct {
 	// The error routes work exactly like the normal routes.
 	Errors *HTTPErrorConfig `json:"errors,omitempty"`
 
+	// NamedRoutes describes a mapping of reusable routes that can be
+	// invoked by their name. This can be used to optimize memory usage
+	// when the same route is needed for many subroutes, by having
+	// the handlers and matchers be only provisioned once, but used from
+	// many places. These routes are not executed unless they are invoked
+	// from another route.
+	//
+	// EXPERIMENTAL: Subject to change or removal.
+	NamedRoutes map[string]*Route `json:"named_routes,omitempty"`
+
 	// How to handle TLS connections. At least one policy is
 	// required to enable HTTPS on this server if automatic
 	// HTTPS is disabled or does not apply.