ci: deflake integration tests (#3966)

* ci: deflake integration tests by pulling Caddy for the running config until new config is loaded

caddytest/caddytest.go

@@ -14,6 +14,7 @@ import (
 	"net/http/cookiejar"
 	"os"
 	"path"
+	"reflect"
 	"regexp"
 	"runtime"
 	"strings"
@@ -98,6 +99,10 @@ func (tc *Tester) InitServer(rawConfig string, configType string) {
 		tc.t.Logf("failed to load config: %s", err)
 		tc.t.Fail()
 	}
+	if err := tc.ensureConfigRunning(rawConfig, configType); err != nil {
+		tc.t.Logf("failed ensurng config is running: %s", err)
+		tc.t.Fail()
+	}
 }
 
 // InitServer this will configure the server with a configurion of a specific
@@ -171,20 +176,57 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 	return nil
 }
 
-var hasValidated bool
+func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error {
-var arePrerequisitesValid bool
+	expectedBytes := []byte(prependCaddyFilePath(rawConfig))
-
+	if configType != "json" {
-func validateTestPrerequisites() error {
+		adapter := caddyconfig.GetAdapter(configType)
-
+		if adapter == nil {
-	if hasValidated {
+			return fmt.Errorf("adapter of config type is missing: %s", configType)
-		if !arePrerequisitesValid {
-			return errors.New("caddy integration prerequisites failed. see first error")
 		}
-		return nil
+		expectedBytes, _, _ = adapter.Adapt([]byte(rawConfig), nil)
 	}
 
-	hasValidated = true
+	var expected interface{}
-	arePrerequisitesValid = false
+	err := json.Unmarshal(expectedBytes, &expected)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{
+		Timeout: Default.LoadRequestTimeout,
+	}
+
+	fetchConfig := func(client *http.Client) interface{} {
+		resp, err := client.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
+		if err != nil {
+			return nil
+		}
+		defer resp.Body.Close()
+		actualBytes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil
+		}
+		var actual interface{}
+		err = json.Unmarshal(actualBytes, &actual)
+		if err != nil {
+			return nil
+		}
+		return actual
+	}
+
+	for retries := 4; retries > 0; retries-- {
+		if reflect.DeepEqual(expected, fetchConfig(client)) {
+			return nil
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	tc.t.Errorf("POSTed configuration isn't active")
+	return errors.New("EnsureConfigRunning: POSTed configuration isn't active")
+}
+
+// validateTestPrerequisites ensures the certificates are available in the
+// designated path and Caddy sub-process is running.
+func validateTestPrerequisites() error {
 
 	// check certificates are found
 	for _, certName := range Default.Certifcates {
@@ -200,20 +242,14 @@ func validateTestPrerequisites() error {
 			caddycmd.Main()
 		}()
 
-		// wait for caddy to start
+		// wait for caddy to start serving the initial config
-		retries := 4
+		for retries := 4; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
-		for ; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
 			time.Sleep(10 * time.Millisecond)
 		}
 	}
 
-	// assert that caddy is running
+	// one more time to return the error
-	if err := isCaddyAdminRunning(); err != nil {
+	return isCaddyAdminRunning()
-		return err
-	}
-
-	arePrerequisitesValid = true
-	return nil
 }
 
 func isCaddyAdminRunning() error {