diff --git a/caddyconfig/httpcaddyfile/directives.go b/caddyconfig/httpcaddyfile/directives.go
index 360f91e7..9da205e2 100644
--- a/caddyconfig/httpcaddyfile/directives.go
+++ b/caddyconfig/httpcaddyfile/directives.go
@@ -485,7 +485,7 @@ func (sb serverBlock) hostsFromKeysNotHTTP(httpPort string) []string {
 		if addr.Host == "" {
 			continue
 		}
-		if addr.Scheme != "http" && addr.Port != httpPort {
+		if addr.Scheme != "http" || addr.Port != httpPort {
 			hostMap[addr.Host] = struct{}{}
 		}
 	}
diff --git a/caddytest/integration/caddyfile_adapt/tls_automation_policies_8.txt b/caddytest/integration/caddyfile_adapt/tls_automation_policies_8.txt
new file mode 100644
index 00000000..1703178e
--- /dev/null
+++ b/caddytest/integration/caddyfile_adapt/tls_automation_policies_8.txt
@@ -0,0 +1,98 @@
+# (this Caddyfile is contrived, but based on issues #4176 and #4198)
+
+http://example.com {
+}
+
+https://example.com {
+	tls abc@example.com
+}
+
+http://localhost:8081 {
+}
+
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":443"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv1": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"example.com"
+									]
+								}
+							],
+							"terminal": true
+						}
+					]
+				},
+				"srv2": {
+					"listen": [
+						":8081"
+					],
+					"routes": [
+						{
+							"match": [
+								{
+									"host": [
+										"localhost"
+									]
+								}
+							],
+							"terminal": true
+						}
+					],
+					"automatic_https": {
+						"skip": [
+							"localhost"
+						]
+					}
+				}
+			}
+		},
+		"tls": {
+			"automation": {
+				"policies": [
+					{
+						"subjects": [
+							"example.com"
+						],
+						"issuers": [
+							{
+								"email": "abc@example.com",
+								"module": "acme"
+							},
+							{
+								"email": "abc@example.com",
+								"module": "zerossl"
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}
\ No newline at end of file