diff --git a/caddy.go b/caddy.go
index 7dd989c9..9aba97fa 100644
--- a/caddy.go
+++ b/caddy.go
@@ -20,6 +20,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/fs"
@@ -778,7 +779,10 @@ func exitProcess(ctx context.Context, logger *zap.Logger) {
 			} else {
 				logger.Error("unclean shutdown")
 			}
-			os.Exit(exitCode)
+			// check if we are in test environment, and dont call exit if we are
+			if flag.Lookup("test.v") == nil || strings.Contains(os.Args[0], ".test") {
+				os.Exit(exitCode)
+			}
 		}()
 
 		if remoteAdminServer != nil {
diff --git a/caddytest/caddytest.go b/caddytest/caddytest.go
index 05aa1e3f..d4a0a6c8 100644
--- a/caddytest/caddytest.go
+++ b/caddytest/caddytest.go
@@ -81,6 +81,10 @@ func NewTester(t testing.TB) *Tester {
 	}
 }
 
+func (t *Tester) T() testing.TB {
+	return t.t
+}
+
 type configLoadError struct {
 	Response string
 }
@@ -92,33 +96,37 @@ func timeElapsed(start time.Time, name string) {
 	log.Printf("%s took %s", name, elapsed)
 }
 
-// InitServer this will configure the server with a configurion of a specific
-// type. The configType must be either "json" or the adapter type.
-func (tc *Tester) InitServer(rawConfig string, configType string) {
-	if err := tc.initServer(rawConfig, configType); err != nil {
-		tc.t.Logf("failed to load config: %s", err)
+// launch caddy will start the server
+func (tc *Tester) LaunchCaddy() {
+	if err := tc.startServer(); err != nil {
+		tc.t.Logf("failed to start server: %s", err)
 		tc.t.Fail()
 	}
-	if err := tc.ensureConfigRunning(rawConfig, configType); err != nil {
+}
+
+// DEPRECATED: InitServer
+// Initserver is shorthand for LaunchCaddy() and LoadConfig(rawConfig, configType string)
+func (tc *Tester) InitServer(rawConfig string, configType string) {
+	if err := tc.startServer(); err != nil {
+		tc.t.Logf("failed to start server: %s", err)
+		tc.t.Fail()
+	}
+	if err := tc.LoadConfig(rawConfig, configType); err != nil {
 		tc.t.Logf("failed ensuring config is running: %s", err)
 		tc.t.Fail()
 	}
 }
 
-// InitServer this will configure the server with a configurion of a specific
-// type. The configType must be either "json" or the adapter type.
-func (tc *Tester) initServer(rawConfig string, configType string) error {
+func (tc *Tester) startServer() error {
 	if testing.Short() {
 		tc.t.SkipNow()
 		return nil
 	}
-
-	err := validateTestPrerequisites(tc.t)
+	err := validateAndStartServer(tc.t)
 	if err != nil {
 		tc.t.Skipf("skipping tests as failed integration prerequisites. %s", err)
 		return nil
 	}
-
 	tc.t.Cleanup(func() {
 		if tc.t.Failed() && tc.configLoaded {
 			res, err := http.Get(fmt.Sprintf("http://localhost:%d/config/", Default.AdminPort))
@@ -133,8 +141,35 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 			_ = json.Indent(&out, body, "", "  ")
 			tc.t.Logf("----------- failed with config -----------\n%s", out.String())
 		}
-	})
+		// now shutdown the server, since the test is done.
 
+		_, err := http.Post(fmt.Sprintf("http://localhost:%d/stop", Default.AdminPort), "", nil)
+		if err != nil {
+			tc.t.Log("couldn't stop admin server")
+		}
+		time.Sleep(1 * time.Millisecond)
+		// try ensure the admin api is stopped three times.
+		for retries := 0; retries < 3; retries++ {
+			if isCaddyAdminRunning() != nil {
+				return
+			}
+			time.Sleep(10 * time.Millisecond)
+			tc.t.Log("timed out waiting for admin server to stop")
+		}
+	})
+	return nil
+}
+
+func (tc *Tester) MustLoadConfig(rawConfig string, configType string) {
+	if err := tc.LoadConfig(rawConfig, configType); err != nil {
+		tc.t.Logf("failed ensuring config is running: %s", err)
+		tc.t.Fail()
+	}
+}
+
+// LoadConfig loads the config to the tester server and also ensures that the config was loaded
+func (tc *Tester) LoadConfig(rawConfig string, configType string) error {
+	originalRawConfig := rawConfig
 	rawConfig = prependCaddyFilePath(rawConfig)
 	// normalize JSON config
 	if configType == "json" {
@@ -185,7 +220,7 @@ func (tc *Tester) initServer(rawConfig string, configType string) error {
 	}
 
 	tc.configLoaded = true
-	return nil
+	return tc.ensureConfigRunning(originalRawConfig, configType)
 }
 
 func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error {
@@ -226,7 +261,9 @@ func (tc *Tester) ensureConfigRunning(rawConfig string, configType string) error
 		return actual
 	}
 
-	for retries := 10; retries > 0; retries-- {
+	// TODO: does this really need to be tried more than once?
+	// Caddy should block until the new config is loaded, which means needing to wait is a caddy bug
+	for retries := 3; retries > 0; retries-- {
 		if reflect.DeepEqual(expected, fetchConfig(client)) {
 			return nil
 		}
@@ -241,9 +278,9 @@ const initConfig = `{
 }
 `
 
-// validateTestPrerequisites ensures the certificates are available in the
-// designated path and Caddy sub-process is running.
-func validateTestPrerequisites(t testing.TB) error {
+// validateAndStartServer ensures the certificates are available in the
+// designated path, launches caddy, and then ensures the Caddy sub-process is running.
+func validateAndStartServer(t testing.TB) error {
 	// check certificates are found
 	for _, certName := range Default.Certificates {
 		if _, err := os.Stat(getIntegrationDir() + certName); errors.Is(err, fs.ErrNotExist) {
@@ -269,9 +306,8 @@ func validateTestPrerequisites(t testing.TB) error {
 		go func() {
 			caddycmd.Main()
 		}()
-
-		// wait for caddy to start serving the initial config
-		for retries := 10; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
+		// wait for caddy admin api to start. it should happen quickly.
+		for retries := 3; retries > 0 && isCaddyAdminRunning() != nil; retries-- {
 			time.Sleep(1 * time.Second)
 		}
 	}
@@ -342,8 +378,8 @@ func CreateTestingTransport() *http.Transport {
 // AssertLoadError will load a config and expect an error
 func AssertLoadError(t *testing.T, rawConfig string, configType string, expectedError string) {
 	tc := NewTester(t)
-
-	err := tc.initServer(rawConfig, configType)
+	tc.LaunchCaddy()
+	err := tc.LoadConfig(rawConfig, configType)
 	if !strings.Contains(err.Error(), expectedError) {
 		t.Errorf("expected error \"%s\" but got \"%s\"", expectedError, err.Error())
 	}
diff --git a/caddytest/integration/acme_test.go b/caddytest/integration/acme_test.go
index ceacd1db..43f99b15 100644
--- a/caddytest/integration/acme_test.go
+++ b/caddytest/integration/acme_test.go
@@ -19,19 +19,27 @@ import (
 	"go.uber.org/zap"
 )
 
+func curry2[A, B any](fn func(A, B)) func(a A) func(b B) {
+	return func(a A) func(B) {
+		return func(b B) {
+			fn(a, b)
+		}
+	}
+}
+
 const acmeChallengePort = 9081
 
-// Test the basic functionality of Caddy's ACME server
-func TestACMEServerWithDefaults(t *testing.T) {
-	ctx := context.Background()
-	logger, err := zap.NewDevelopment()
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
+func TestAcmeServer(t *testing.T) {
 	tester := caddytest.NewTester(t)
-	tester.InitServer(`
+	tester.LaunchCaddy()
+	t.Run("WithDefaults", curry2(testACMEServerWithDefaults)(tester))
+	t.Run("WithMismatchedChallenges", curry2(testACMEServerWithDefaults)(tester))
+}
+
+// Test the basic functionality of Caddy's ACME server
+func testACMEServerWithDefaults(tester *caddytest.Tester, t *testing.T) {
+	ctx := context.Background()
+	tester.MustLoadConfig(`
 	{
 		skip_install_trust
 		admin localhost:2999
@@ -44,6 +52,7 @@ func TestACMEServerWithDefaults(t *testing.T) {
 	}
   `, "caddyfile")
 
+	logger := caddy.Log().Named("acmeserver")
 	client := acmez.Client{
 		Client: &acme.Client{
 			Directory:  "https://acme.localhost:9443/acme/local/directory",
@@ -93,20 +102,10 @@ func TestACMEServerWithDefaults(t *testing.T) {
 	}
 }
 
-func TestACMEServerWithMismatchedChallenges(t *testing.T) {
+func testACMEServerWithMismatchedChallenges(tester *caddytest.Tester, t *testing.T) {
 	ctx := context.Background()
 	logger := caddy.Log().Named("acmez")
-
-	tester := caddytest.NewTester(t)
-	tester.InitServer(`
-	{
-		skip_install_trust
-		admin localhost:2999
-		http_port     9080
-		https_port    9443
-		local_certs
-	}
-	acme.localhost {
+	tester.MustLoadConfig(`
 		acme_server {
 			challenges tls-alpn-01
 		}
diff --git a/caddytest/integration/acmeserver_test.go b/caddytest/integration/acmeserver_test.go
index 22b716f8..f05fb836 100644
--- a/caddytest/integration/acmeserver_test.go
+++ b/caddytest/integration/acmeserver_test.go
@@ -8,10 +8,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddytest"
 	"github.com/mholt/acmez/v2"
 	"github.com/mholt/acmez/v2/acme"
-	"go.uber.org/zap"
 )
 
 func TestACMEServerDirectory(t *testing.T) {
@@ -66,11 +66,7 @@ func TestACMEServerAllowPolicy(t *testing.T) {
   `, "caddyfile")
 
 	ctx := context.Background()
-	logger, err := zap.NewDevelopment()
-	if err != nil {
-		t.Error(err)
-		return
-	}
+	logger := caddy.Log().Named("acmez")
 
 	client := acmez.Client{
 		Client: &acme.Client{
@@ -155,11 +151,7 @@ func TestACMEServerDenyPolicy(t *testing.T) {
   `, "caddyfile")
 
 	ctx := context.Background()
-	logger, err := zap.NewDevelopment()
-	if err != nil {
-		t.Error(err)
-		return
-	}
+	logger := caddy.Log().Named("acmez")
 
 	client := acmez.Client{
 		Client: &acme.Client{
@@ -197,7 +189,7 @@ func TestACMEServerDenyPolicy(t *testing.T) {
 		_, err := client.ObtainCertificateForSANs(ctx, account, certPrivateKey, []string{"deny.localhost"})
 		if err == nil {
 			t.Errorf("obtaining certificate for 'deny.localhost' domain")
-		} else if err != nil && !strings.Contains(err.Error(), "urn:ietf:params:acme:error:rejectedIdentifier") {
+		} else if !strings.Contains(err.Error(), "urn:ietf:params:acme:error:rejectedIdentifier") {
 			t.Logf("unexpected error: %v", err)
 		}
 	}
diff --git a/caddytest/integration/stream_test.go b/caddytest/integration/stream_test.go
index d2f2fd79..d82882d6 100644
--- a/caddytest/integration/stream_test.go
+++ b/caddytest/integration/stream_test.go
@@ -21,7 +21,7 @@ import (
 // (see https://github.com/caddyserver/caddy/issues/3556 for use case)
 func TestH2ToH2CStream(t *testing.T) {
 	tester := caddytest.NewTester(t)
-	tester.InitServer(` 
+	tester.InitServer(`
   {
 	"admin": {
 		"listen": "localhost:2999"
@@ -205,18 +205,11 @@ func testH2ToH2CStreamServeH2C(t *testing.T) *http.Server {
 // (see https://github.com/caddyserver/caddy/issues/3606 for use case)
 func TestH2ToH1ChunkedResponse(t *testing.T) {
 	tester := caddytest.NewTester(t)
-	tester.InitServer(` 
+	tester.InitServer(`
 {
 	"admin": {
 		"listen": "localhost:2999"
 	},
-  "logging": {
-    "logs": {
-      "default": {
-        "level": "DEBUG"
-      }
-    }
-  },
   "apps": {
     "http": {
       "http_port": 9080,
diff --git a/logging.go b/logging.go
index ca10beee..09bf8f84 100644
--- a/logging.go
+++ b/logging.go
@@ -16,6 +16,7 @@ package caddy
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"io"
 	"log"
@@ -699,7 +700,13 @@ type defaultCustomLog struct {
 // and enables INFO-level logs and higher.
 func newDefaultProductionLog() (*defaultCustomLog, error) {
 	cl := new(CustomLog)
-	cl.writerOpener = StderrWriter{}
+	f := flag.Lookup("test.v")
+	if (f != nil && f.Value.String() != "true") || strings.Contains(os.Args[0], ".test") {
+		cl.writerOpener = &DiscardWriter{}
+	} else {
+		cl.writerOpener = StderrWriter{}
+	}
+
 	var err error
 	cl.writer, err = cl.writerOpener.OpenWriter()
 	if err != nil {