caddyfile: Preprocess env vars in {$THIS} format (#2963)

* transform a caddyfile with environment variables

* support adapt time and runtime variables in the caddyfile

* caddyfile: Pre-process environment variables before parsing

Co-authored-by: Matt Holt <mholt@users.noreply.github.com>

caddyconfig/caddyfile/adapter.go

@@ -15,7 +15,6 @@
 package caddyfile
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 
@@ -42,7 +41,7 @@ func (a Adapter) Adapt(body []byte, options map[string]interface{}) ([]byte, []c
 		filename = "Caddyfile"
 	}
 
-	serverBlocks, err := Parse(filename, bytes.NewReader(body))
+	serverBlocks, err := Parse(filename, body)
 	if err != nil {
 		return nil, nil, err
 	}

caddyconfig/caddyfile/parse.go

@@ -15,6 +15,7 @@
 package caddyfile
 
 import (
+	"bytes"
 	"io"
 	"log"
 	"os"
@@ -28,8 +29,12 @@ import (
 // Directives that do not appear in validDirectives will cause
 // an error. If you do not want to check for valid directives,
 // pass in nil instead.
-func Parse(filename string, input io.Reader) ([]ServerBlock, error) {
-	tokens, err := allTokens(filename, input)
+//
+// Environment variables in {$ENVIRONMENT_VARIABLE} notation
+// will be replaced before parsing begins.
+func Parse(filename string, input []byte) ([]ServerBlock, error) {
+	input = replaceEnvVars(input)
+	tokens, err := allTokens(filename, bytes.NewReader(input))
 	if err != nil {
 		return nil, err
 	}
@@ -37,6 +42,41 @@ func Parse(filename string, input io.Reader) ([]ServerBlock, error) {
 	return p.parseAll()
 }
 
+// replaceEnvVars replaces all occurrences of environment variables.
+func replaceEnvVars(input []byte) []byte {
+	var offset int
+	for {
+		begin := bytes.Index(input[offset:], spanOpen)
+		if begin < 0 {
+			break
+		}
+		begin += offset // make beginning relative to input, not offset
+		end := bytes.Index(input[begin+len(spanOpen):], spanClose)
+		if end < 0 {
+			break
+		}
+		end += begin + len(spanOpen) // make end relative to input, not begin
+
+		// get the name; if there is no name, skip it
+		envVarName := input[begin+len(spanOpen) : end]
+		if len(envVarName) == 0 {
+			offset = end + len(spanClose)
+			continue
+		}
+
+		// get the value of the environment variable
+		envVarValue := []byte(os.Getenv(string(envVarName)))
+
+		// splice in the value
+		input = append(input[:begin],
+			append(envVarValue, input[end+len(spanClose):]...)...)
+
+		// continue at the end of the replacement
+		offset = begin + len(envVarValue)
+	}
+	return input
+}
+
 // allTokens lexes the entire input, but does not parse it.
 // It returns all the tokens from the input, unstructured
 // and in order.
@@ -128,7 +168,7 @@ func (p *parser) addresses() error {
 	var expectingAnother bool
 
 	for {
-		tkn := replaceEnvVars(p.Val())
+		tkn := p.Val()
 
 		// special case: import directive replaces tokens during parse-time
 		if tkn == "import" && p.isNewLine() {
@@ -245,7 +285,7 @@ func (p *parser) doImport() error {
 	if !p.NextArg() {
 		return p.ArgErr()
 	}
-	importPattern := replaceEnvVars(p.Val())
+	importPattern := p.Val()
 	if importPattern == "" {
 		return p.Err("Import requires a non-empty filepath")
 	}
@@ -353,8 +393,6 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {
 // are loaded into the current server block for later use
 // by directive setup functions.
 func (p *parser) directive() error {
-	// evaluate any env vars in directive token
-	p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text)
 
 	// a segment is a list of tokens associated with this directive
 	var segment Segment
@@ -379,7 +417,7 @@ func (p *parser) directive() error {
 			p.cursor-- // cursor is advanced when we continue, so roll back one more
 			continue
 		}
-		p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text)
+
 		segment = append(segment, p.Token())
 	}
 
@@ -414,36 +452,6 @@ func (p *parser) closeCurlyBrace() error {
 	return nil
 }
 
-// replaceEnvVars replaces environment variables that appear in the token
-// and understands both the $UNIX and %WINDOWS% syntaxes.
-func replaceEnvVars(s string) string {
-	s = replaceEnvReferences(s, "{%", "%}")
-	s = replaceEnvReferences(s, "{$", "}")
-	return s
-}
-
-// replaceEnvReferences performs the actual replacement of env variables
-// in s, given the placeholder start and placeholder end strings.
-func replaceEnvReferences(s, refStart, refEnd string) string {
-	index := strings.Index(s, refStart)
-	for index != -1 {
-		endIndex := strings.Index(s[index:], refEnd)
-		if endIndex == -1 {
-			break
-		}
-
-		endIndex += index
-		if endIndex > index+len(refStart) {
-			ref := s[index : endIndex+len(refEnd)]
-			s = strings.Replace(s, ref, os.Getenv(ref[len(refStart):len(ref)-len(refEnd)]), -1)
-		} else {
-			return s
-		}
-		index = strings.Index(s, refStart)
-	}
-	return s
-}
-
 func (p *parser) isSnippet() (bool, string) {
 	keys := p.block.Keys
 	// A snippet block is a single key with parens. Nothing else qualifies.
@@ -514,3 +522,7 @@ func (s Segment) Directive() string {
 	}
 	return ""
 }
+
+// spanOpen and spanClose are used to bound spans that
+// contain the name of an environment variable.
+var spanOpen, spanClose = []byte{'{', '$'}, []byte{'}'}

caddyconfig/caddyfile/parse_test.go

@@ -15,6 +15,7 @@
 package caddyfile
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -473,89 +474,81 @@ func TestParseAll(t *testing.T) {
 }
 
 func TestEnvironmentReplacement(t *testing.T) {
-	os.Setenv("PORT", "8080")
-	os.Setenv("ADDRESS", "servername.com")
 	os.Setenv("FOOBAR", "foobar")
-	os.Setenv("PARTIAL_DIR", "r1")
 
-	// basic test; unix-style env vars
-	p := testParser(`{$ADDRESS}`)
-	blocks, _ := p.parseAll()
-	if actual, expected := blocks[0].Keys[0], "servername.com"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-
-	// basic test; unix-style env vars
-	p = testParser(`di{$PARTIAL_DIR}`)
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Keys[0], "dir1"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-
-	// multiple vars per token
-	p = testParser(`{$ADDRESS}:{$PORT}`)
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Keys[0], "servername.com:8080"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-
-	// windows-style var and unix style in same token
-	p = testParser(`{%ADDRESS%}:{$PORT}`)
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Keys[0], "servername.com:8080"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-
-	// reverse order
-	p = testParser(`{$ADDRESS}:{%PORT%}`)
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Keys[0], "servername.com:8080"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-
-	// env var in server block body as argument
-	p = testParser(":{%PORT%}\ndir1 {$FOOBAR}")
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Keys[0], ":8080"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-	if actual, expected := blocks[0].Segments[0][1].Text, "foobar"; expected != actual {
-		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
-	}
-
-	// combined windows env vars in argument
-	p = testParser(":{%PORT%}\ndir1 {%ADDRESS%}/{%FOOBAR%}")
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Segments[0][1].Text, "servername.com/foobar"; expected != actual {
-		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
-	}
-
-	// malformed env var (windows)
-	p = testParser(":1234\ndir1 {%ADDRESS}")
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Segments[0][1].Text, "{%ADDRESS}"; expected != actual {
-		t.Errorf("Expected host to be '%s' but was '%s'", expected, actual)
-	}
-
-	// malformed (non-existent) env var (unix)
-	p = testParser(`:{$PORT$}`)
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Keys[0], ":"; expected != actual {
-		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
-	}
-
-	// in quoted field
-	p = testParser(":1234\ndir1 \"Test {$FOOBAR} test\"")
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Segments[0][1].Text, "Test foobar test"; expected != actual {
-		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
-	}
-
-	// after end token
-	p = testParser(":1234\nanswer \"{{ .Name }} {$FOOBAR}\"")
-	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Segments[0][1].Text, "{{ .Name }} foobar"; expected != actual {
-		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
+	for i, test := range []struct {
+		input  string
+		expect string
+	}{
+		{
+			input:  "",
+			expect: "",
+		},
+		{
+			input:  "foo",
+			expect: "foo",
+		},
+		{
+			input:  "{$NOT_SET}",
+			expect: "",
+		},
+		{
+			input:  "foo{$NOT_SET}bar",
+			expect: "foobar",
+		},
+		{
+			input:  "{$FOOBAR}",
+			expect: "foobar",
+		},
+		{
+			input:  "foo {$FOOBAR} bar",
+			expect: "foo foobar bar",
+		},
+		{
+			input:  "foo{$FOOBAR}bar",
+			expect: "foofoobarbar",
+		},
+		{
+			input:  "foo\n{$FOOBAR}\nbar",
+			expect: "foo\nfoobar\nbar",
+		},
+		{
+			input:  "{$FOOBAR} {$FOOBAR}",
+			expect: "foobar foobar",
+		},
+		{
+			input:  "{$FOOBAR}{$FOOBAR}",
+			expect: "foobarfoobar",
+		},
+		{
+			input:  "{$FOOBAR",
+			expect: "{$FOOBAR",
+		},
+		{
+			input:  "{$LONGER_NAME $FOOBAR}",
+			expect: "",
+		},
+		{
+			input:  "{$}",
+			expect: "{$}",
+		},
+		{
+			input:  "{$$}",
+			expect: "",
+		},
+		{
+			input:  "{$",
+			expect: "{$",
+		},
+		{
+			input:  "}{$",
+			expect: "}{$",
+		},
+	} {
+		actual := replaceEnvVars([]byte(test.input))
+		if !bytes.Equal(actual, []byte(test.expect)) {
+			t.Errorf("Test %d: Expected: '%s' but got '%s'", i, test.expect, actual)
+		}
 	}
 }