Merge branch 'master' into acme-database

.editorconfig

@@ -1,5 +1,5 @@
 [*]
 end_of_line = lf
 
-[caddytest/integration/caddyfile_adapt/*.txt]
+[caddytest/integration/caddyfile_adapt/*.caddyfiletest]
 indent_style = tab

caddyconfig/caddyfile/formatter.go

@@ -124,18 +124,22 @@ func Format(input []byte) []byte {
 		}
 		// if we're in a heredoc, all characters are read&write as-is
 		if heredoc == heredocOpened {
-			write(ch)
 			heredocClosingMarker = append(heredocClosingMarker, ch)
-			if len(heredocClosingMarker) > len(heredocMarker) {
+			if len(heredocClosingMarker) > len(heredocMarker)+1 { // We assert that the heredocClosingMarker is followed by a unicode.Space
 				heredocClosingMarker = heredocClosingMarker[1:]
 			}
 			// check if we're done
-			if slices.Equal(heredocClosingMarker, heredocMarker) {
+			if unicode.IsSpace(ch) && slices.Equal(heredocClosingMarker[:len(heredocClosingMarker)-1], heredocMarker) {
 				heredocMarker = nil
 				heredocClosingMarker = nil
 				heredoc = heredocClosed
+			} else {
+				write(ch)
+				if ch == '\n' {
+					heredocClosingMarker = heredocClosingMarker[:0]
+				}
+				continue
 			}
-			continue
 		}
 
 		if last == '<' && space {

caddyconfig/caddyfile/parse.go

@@ -160,14 +160,14 @@ func (p *parser) begin() error {
 	}
 
 	if ok, name := p.isNamedRoute(); ok {
-		// named routes only have one key, the route name
-		p.block.Keys = []string{name}
-		p.block.IsNamedRoute = true
-
 		// we just need a dummy leading token to ease parsing later
 		nameToken := p.Token()
 		nameToken.Text = name
 
+		// named routes only have one key, the route name
+		p.block.Keys = []Token{nameToken}
+		p.block.IsNamedRoute = true
+
 		// get all the tokens from the block, including the braces
 		tokens, err := p.blockTokens(true)
 		if err != nil {
@@ -211,10 +211,11 @@ func (p *parser) addresses() error {
 	var expectingAnother bool
 
 	for {
-		tkn := p.Val()
+		value := p.Val()
+		token := p.Token()
 
 		// special case: import directive replaces tokens during parse-time
-		if tkn == "import" && p.isNewLine() {
+		if value == "import" && p.isNewLine() {
 			err := p.doImport(0)
 			if err != nil {
 				return err
@@ -223,9 +224,9 @@ func (p *parser) addresses() error {
 		}
 
 		// Open brace definitely indicates end of addresses
-		if tkn == "{" {
+		if value == "{" {
 			if expectingAnother {
-				return p.Errf("Expected another address but had '%s' - check for extra comma", tkn)
+				return p.Errf("Expected another address but had '%s' - check for extra comma", value)
 			}
 			// Mark this server block as being defined with braces.
 			// This is used to provide a better error message when
@@ -237,15 +238,15 @@ func (p *parser) addresses() error {
 		}
 
 		// Users commonly forget to place a space between the address and the '{'
-		if strings.HasSuffix(tkn, "{") {
+		if strings.HasSuffix(value, "{") {
-			return p.Errf("Site addresses cannot end with a curly brace: '%s' - put a space between the token and the brace", tkn)
+			return p.Errf("Site addresses cannot end with a curly brace: '%s' - put a space between the token and the brace", value)
 		}
 
-		if tkn != "" { // empty token possible if user typed ""
+		if value != "" { // empty token possible if user typed ""
 			// Trailing comma indicates another address will follow, which
 			// may possibly be on the next line
-			if tkn[len(tkn)-1] == ',' {
+			if value[len(value)-1] == ',' {
-				tkn = tkn[:len(tkn)-1]
+				value = value[:len(value)-1]
 				expectingAnother = true
 			} else {
 				expectingAnother = false // but we may still see another one on this line
@@ -254,11 +255,12 @@ func (p *parser) addresses() error {
 			// If there's a comma here, it's probably because they didn't use a space
 			// between their two domains, e.g. "foo.com,bar.com", which would not be
 			// parsed as two separate site addresses.
-			if strings.Contains(tkn, ",") {
+			if strings.Contains(value, ",") {
-				return p.Errf("Site addresses cannot contain a comma ',': '%s' - put a space after the comma to separate site addresses", tkn)
+				return p.Errf("Site addresses cannot contain a comma ',': '%s' - put a space after the comma to separate site addresses", value)
 			}
 
-			p.block.Keys = append(p.block.Keys, tkn)
+			token.Text = value
+			p.block.Keys = append(p.block.Keys, token)
 		}
 
 		// Advance token and possibly break out of loop or return error
@@ -637,8 +639,8 @@ func (p *parser) closeCurlyBrace() error {
 func (p *parser) isNamedRoute() (bool, string) {
 	keys := p.block.Keys
 	// A named route block is a single key with parens, prefixed with &.
-	if len(keys) == 1 && strings.HasPrefix(keys[0], "&(") && strings.HasSuffix(keys[0], ")") {
+	if len(keys) == 1 && strings.HasPrefix(keys[0].Text, "&(") && strings.HasSuffix(keys[0].Text, ")") {
-		return true, strings.TrimSuffix(keys[0][2:], ")")
+		return true, strings.TrimSuffix(keys[0].Text[2:], ")")
 	}
 	return false, ""
 }
@@ -646,8 +648,8 @@ func (p *parser) isNamedRoute() (bool, string) {
 func (p *parser) isSnippet() (bool, string) {
 	keys := p.block.Keys
 	// A snippet block is a single key with parens. Nothing else qualifies.
-	if len(keys) == 1 && strings.HasPrefix(keys[0], "(") && strings.HasSuffix(keys[0], ")") {
+	if len(keys) == 1 && strings.HasPrefix(keys[0].Text, "(") && strings.HasSuffix(keys[0].Text, ")") {
-		return true, strings.TrimSuffix(keys[0][1:], ")")
+		return true, strings.TrimSuffix(keys[0].Text[1:], ")")
 	}
 	return false, ""
 }
@@ -691,11 +693,19 @@ func (p *parser) blockTokens(retainCurlies bool) ([]Token, error) {
 // grouped by segments.
 type ServerBlock struct {
 	HasBraces    bool
-	Keys         []string
+	Keys         []Token
 	Segments     []Segment
 	IsNamedRoute bool
 }
 
+func (sb ServerBlock) GetKeysText() []string {
+	res := []string{}
+	for _, k := range sb.Keys {
+		res = append(res, k.Text)
+	}
+	return res
+}
+
 // DispenseDirective returns a dispenser that contains
 // all the tokens in the server block.
 func (sb ServerBlock) DispenseDirective(dir string) *Dispenser {

caddyconfig/caddyfile/parse_test.go

@@ -347,7 +347,7 @@ func TestParseOneAndImport(t *testing.T) {
 				i, len(test.keys), len(result.Keys))
 			continue
 		}
-		for j, addr := range result.Keys {
+		for j, addr := range result.GetKeysText() {
 			if addr != test.keys[j] {
 				t.Errorf("Test %d, key %d: Expected '%s', but was '%s'",
 					i, j, test.keys[j], addr)
@@ -379,8 +379,9 @@ func TestRecursiveImport(t *testing.T) {
 	}
 
 	isExpected := func(got ServerBlock) bool {
-		if len(got.Keys) != 1 || got.Keys[0] != "localhost" {
+		textKeys := got.GetKeysText()
-			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)
+		if len(textKeys) != 1 || textKeys[0] != "localhost" {
+			t.Errorf("got keys unexpected: expect localhost, got %v", textKeys)
 			return false
 		}
 		if len(got.Segments) != 2 {
@@ -474,8 +475,9 @@ func TestDirectiveImport(t *testing.T) {
 	}
 
 	isExpected := func(got ServerBlock) bool {
-		if len(got.Keys) != 1 || got.Keys[0] != "localhost" {
+		textKeys := got.GetKeysText()
-			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)
+		if len(textKeys) != 1 || textKeys[0] != "localhost" {
+			t.Errorf("got keys unexpected: expect localhost, got %v", textKeys)
 			return false
 		}
 		if len(got.Segments) != 2 {
@@ -616,7 +618,7 @@ func TestParseAll(t *testing.T) {
 					i, len(test.keys[j]), j, len(block.Keys))
 				continue
 			}
-			for k, addr := range block.Keys {
+			for k, addr := range block.GetKeysText() {
 				if addr != test.keys[j][k] {
 					t.Errorf("Test %d, block %d, key %d: Expected '%s', but got '%s'",
 						i, j, k, test.keys[j][k], addr)
@@ -769,7 +771,7 @@ func TestSnippets(t *testing.T) {
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
 	}
-	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {
+	if actual, expected := blocks[0].GetKeysText()[0], "http://example.com"; expected != actual {
 		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)
 	}
 	if len(blocks[0].Segments) != 2 {
@@ -801,7 +803,7 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {
 	fileName := writeStringToTempFileOrDie(t, `
 		http://example.com {
 			# This isn't an import directive, it's just an arg with value 'import'
-			basicauth / import password
+			basic_auth / import password
 		}
 	`)
 	// Parse the root file that imports the other one.
@@ -812,12 +814,12 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {
 	}
 	auth := blocks[0].Segments[0]
 	line := auth[0].Text + " " + auth[1].Text + " " + auth[2].Text + " " + auth[3].Text
-	if line != "basicauth / import password" {
+	if line != "basic_auth / import password" {
 		// Previously, it would be changed to:
-		//   basicauth / import /path/to/test/dir/password
+		//   basic_auth / import /path/to/test/dir/password
 		// referencing a file that (probably) doesn't exist and changing the
 		// password!
-		t.Errorf("Expected basicauth tokens to be 'basicauth / import password' but got %#q", line)
+		t.Errorf("Expected basic_auth tokens to be 'basic_auth / import password' but got %#q", line)
 	}
 }
 
@@ -844,7 +846,7 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
 	}
-	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {
+	if actual, expected := blocks[0].GetKeysText()[0], "http://example.com"; expected != actual {
 		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)
 	}
 	if len(blocks[0].Segments) != 1 {

caddyconfig/httpcaddyfile/addresses.go

@@ -88,15 +88,15 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 		// will be served by them; this has the effect of treating each
 		// key of a server block as its own, but without having to repeat its
 		// contents in cases where multiple keys really can be served together
-		addrToKeys := make(map[string][]string)
+		addrToKeys := make(map[string][]caddyfile.Token)
 		for j, key := range sblock.block.Keys {
 			// a key can have multiple listener addresses if there are multiple
 			// arguments to the 'bind' directive (although they will all have
 			// the same port, since the port is defined by the key or is implicit
 			// through automatic HTTPS)
-			addrs, err := st.listenerAddrsForServerBlockKey(sblock, key, options)
+			addrs, err := st.listenerAddrsForServerBlockKey(sblock, key.Text, options)
 			if err != nil {
-				return nil, fmt.Errorf("server block %d, key %d (%s): determining listener address: %v", i, j, key, err)
+				return nil, fmt.Errorf("server block %d, key %d (%s): determining listener address: %v", i, j, key.Text, err)
 			}
 
 			// associate this key with each listener address it is served on
@@ -122,9 +122,9 @@ func (st *ServerType) mapAddressToServerBlocks(originalServerBlocks []serverBloc
 			// parse keys so that we only have to do it once
 			parsedKeys := make([]Address, 0, len(keys))
 			for _, key := range keys {
-				addr, err := ParseAddress(key)
+				addr, err := ParseAddress(key.Text)
 				if err != nil {
-					return nil, fmt.Errorf("parsing key '%s': %v", key, err)
+					return nil, fmt.Errorf("parsing key '%s': %v", key.Text, err)
 				}
 				parsedKeys = append(parsedKeys, addr.Normalize())
 			}

caddyconfig/httpcaddyfile/directives.go

@@ -58,7 +58,8 @@ var directiveOrder = []string{
 	"try_files",
 
 	// middleware handlers; some wrap responses
-	"basicauth",
+	"basicauth", // TODO: deprecated, renamed to basic_auth
+	"basic_auth",
 	"forward_auth",
 	"request_header",
 	"encode",

caddyconfig/httpcaddyfile/httptype.go

@@ -65,8 +65,11 @@ func (st ServerType) Setup(
 	originalServerBlocks := make([]serverBlock, 0, len(inputServerBlocks))
 	for _, sblock := range inputServerBlocks {
 		for j, k := range sblock.Keys {
-			if j == 0 && strings.HasPrefix(k, "@") {
+			if j == 0 && strings.HasPrefix(k.Text, "@") {
-				return nil, warnings, fmt.Errorf("cannot define a matcher outside of a site block: '%s'", k)
+				return nil, warnings, fmt.Errorf("%s:%d: cannot define a matcher outside of a site block: '%s'", k.File, k.Line, k.Text)
+			}
+			if _, ok := registeredDirectives[k.Text]; ok {
+				return nil, warnings, fmt.Errorf("%s:%d: parsed '%s' as a site address, but it is a known directive; directives must appear in a site block", k.File, k.Line, k.Text)
 			}
 		}
 		originalServerBlocks = append(originalServerBlocks, serverBlock{
@@ -490,7 +493,7 @@ func (ServerType) extractNamedRoutes(
 			route.HandlersRaw = []json.RawMessage{caddyconfig.JSONModuleObject(handler, "handler", subroute.CaddyModule().ID.Name(), h.warnings)}
 		}
 
-		namedRoutes[sb.block.Keys[0]] = &route
+		namedRoutes[sb.block.GetKeysText()[0]] = &route
 	}
 	options["named_routes"] = namedRoutes
 
@@ -528,12 +531,12 @@ func (st *ServerType) serversFromPairings(
 		// address), otherwise their routes will improperly be added
 		// to the same server (see issue #4635)
 		for j, sblock1 := range p.serverBlocks {
-			for _, key := range sblock1.block.Keys {
+			for _, key := range sblock1.block.GetKeysText() {
 				for k, sblock2 := range p.serverBlocks {
 					if k == j {
 						continue
 					}
-					if sliceContains(sblock2.block.Keys, key) {
+					if sliceContains(sblock2.block.GetKeysText(), key) {
 						return nil, fmt.Errorf("ambiguous site definition: %s", key)
 					}
 				}

caddytest/caddytest.go

@@ -60,11 +60,11 @@ var (
 type Tester struct {
 	Client       *http.Client
 	configLoaded bool
-	t            *testing.T
+	t            testing.TB
 }
 
 // NewTester will create a new testing client with an attached cookie jar
-func NewTester(t *testing.T) *Tester {
+func NewTester(t testing.TB) *Tester {
 	jar, err := cookiejar.New(nil)
 	if err != nil {
 		t.Fatalf("failed to create cookiejar: %s", err)
@@ -229,7 +229,7 @@ const initConfig = `{
 
 // validateTestPrerequisites ensures the certificates are available in the
 // designated path and Caddy sub-process is running.
-func validateTestPrerequisites(t *testing.T) error {
+func validateTestPrerequisites(t testing.TB) error {
 	// check certificates are found
 	for _, certName := range Default.Certifcates {
 		if _, err := os.Stat(getIntegrationDir() + certName); errors.Is(err, fs.ErrNotExist) {
@@ -373,7 +373,7 @@ func (tc *Tester) AssertRedirect(requestURI string, expectedToLocation string, e
 }
 
 // CompareAdapt adapts a config and then compares it against an expected result
-func CompareAdapt(t *testing.T, filename, rawConfig string, adapterName string, expectedResponse string) bool {
+func CompareAdapt(t testing.TB, filename, rawConfig string, adapterName string, expectedResponse string) bool {
 	cfgAdapter := caddyconfig.GetAdapter(adapterName)
 	if cfgAdapter == nil {
 		t.Logf("unrecognized config adapter '%s'", adapterName)
@@ -432,7 +432,7 @@ func CompareAdapt(t *testing.T, filename, rawConfig string, adapterName string,
 }
 
 // AssertAdapt adapts a config and then tests it against an expected result
-func AssertAdapt(t *testing.T, rawConfig string, adapterName string, expectedResponse string) {
+func AssertAdapt(t testing.TB, rawConfig string, adapterName string, expectedResponse string) {
 	ok := CompareAdapt(t, "Caddyfile", rawConfig, adapterName, expectedResponse)
 	if !ok {
 		t.Fail()
@@ -441,7 +441,7 @@ func AssertAdapt(t *testing.T, rawConfig string, adapterName string, expectedRes
 
 // Generic request functions
 
-func applyHeaders(t *testing.T, req *http.Request, requestHeaders []string) {
+func applyHeaders(t testing.TB, req *http.Request, requestHeaders []string) {
 	requestContentType := ""
 	for _, requestHeader := range requestHeaders {
 		arr := strings.SplitAfterN(requestHeader, ":", 2)

caddytest/integration/acme_test.go

@@ -23,7 +23,7 @@ import (
 	"go.uber.org/zap"
 )
 
-const acmeChallengePort = 8080
+const acmeChallengePort = 9081
 
 // Test the basic functionality of Caddy's ACME server
 func TestACMEServerWithDefaults(t *testing.T) {

caddytest/integration/caddyfile_adapt/http_valid_directive_like_site_address.caddyfiletest

@@ -0,0 +1,46 @@
+http://handle {
+	file_server
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"handle"
+									]
+								}
+							],
+							"handle": [
+								{
+									"handler": "subroute",
+									"routes": [
+										{
+											"handle": [
+												{
+													"handler": "file_server",
+													"hide": [
+														"./Caddyfile"
+													]
+												}
+											]
+										}
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest

@@ -21,6 +21,7 @@ log {
 				ipv4 24
 				ipv6 32
 			}
+			request>client_ip ip_mask 16 32
 			request>headers>Regexp regexp secret REDACTED
 			request>headers>Hash hash
 		}
@@ -41,6 +42,11 @@ log {
 				},
 				"encoder": {
 					"fields": {
+						"request\u003eclient_ip": {
+							"filter": "ip_mask",
+							"ipv4_cidr": 16,
+							"ipv6_cidr": 32
+						},
 						"request\u003eheaders\u003eAuthorization": {
 							"filter": "replace",
 							"value": "REDACTED"