diff --git a/imapserver/server.go b/imapserver/server.go
index 2d38616..7012549 100644
--- a/imapserver/server.go
+++ b/imapserver/server.go
@@ -52,6 +52,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+    "reflect"
 	"runtime/debug"
 	"slices"
 	"sort"
@@ -727,7 +728,8 @@ func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, x
 		}
 	}()
 
-	if xtls {
+    isAlreadyTLS := reflect.TypeOf(nc) == reflect.TypeFor[*tls.Conn]()
+	if xtls && !isAlreadyTLS {
 		// Start TLS on connection. We perform the handshake explicitly, so we can set a
 		// timeout, do client certificate authentication, log TLS details afterwards.
 		c.xtlsHandshakeAndAuthenticate(c.conn)
diff --git a/smtpserver/server.go b/smtpserver/server.go
index 70b584c..1e49dce 100644
--- a/smtpserver/server.go
+++ b/smtpserver/server.go
@@ -23,6 +23,7 @@ import (
 	"net"
 	"net/textproto"
 	"os"
+    "reflect"
 	"runtime/debug"
 	"slices"
 	"sort"
@@ -893,7 +894,8 @@ func serve(listenerName string, cid int64, hostname dns.Domain, tlsConfig *tls.C
 		}
 	}()
 
-	if xtls {
+    isAlreadyTLS := reflect.TypeOf(nc) == reflect.TypeFor[*tls.Conn]()
+	if xtls && !isAlreadyTLS {
 		// Start TLS on connection. We perform the handshake explicitly, so we can set a
 		// timeout, do client certificate authentication, log TLS details afterwards.
 		c.xtlsHandshakeAndAuthenticate(c.conn)