caddytls: Support multiple issuers (#3862)

* caddytls: Support multiple issuers

Defaults are Let's Encrypt and ZeroSSL.

There are probably bugs.

* Commit updated integration tests, d'oh

* Update go.mod

caddyconfig/httpcaddyfile/builtins.go

@@ -88,7 +88,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var certSelector caddytls.CustomCertSelectionPolicy
 	var acmeIssuer *caddytls.ACMEIssuer
 	var internalIssuer *caddytls.InternalIssuer
-	var issuer certmagic.Issuer
+	var issuers []certmagic.Issuer
 	var onDemand bool
 
 	for h.Next() {
@@ -297,10 +297,11 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				if err != nil {
 					return nil, err
 				}
-				issuer, ok = unm.(certmagic.Issuer)
+				issuer, ok := unm.(certmagic.Issuer)
 				if !ok {
 					return nil, h.Errf("module %s is not a certmagic.Issuer", mod.ID)
 				}
+				issuers = append(issuers, issuer)
 
 			case "dns":
 				if !h.NextArg() {
@@ -371,44 +372,30 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 		})
 	}
 
-	// issuer
+	if len(issuers) > 0 && (acmeIssuer != nil || internalIssuer != nil) {
-	if acmeIssuer != nil && internalIssuer != nil {
+		// some tls subdirectives are shortcuts that implicitly configure issuers, and the
-		// the logic to support this would be complex
+		// user can also configure issuers explicitly using the issuer subdirective; the
-		return nil, h.Err("cannot use both ACME and internal issuers in same server block")
+		// logic to support both would likely be complex, or at least unintuitive
+		return nil, h.Err("cannot mix issuer subdirective (explicit issuers) with other issuer-specific subdirectives (implicit issuers)")
 	}
-	if issuer != nil && (acmeIssuer != nil || internalIssuer != nil) {
+	for _, issuer := range issuers {
-		// similarly, the logic to support this would be complex
-		return nil, h.Err("when defining an issuer, all its config must be in its block, rather than from separate tls subdirectives")
-	}
-	switch {
-	case issuer != nil:
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.cert_issuer",
 			Value: issuer,
 		})
-
+	}
-	case internalIssuer != nil:
+	if acmeIssuer != nil {
-		configVals = append(configVals, ConfigValue{
-			Class: "tls.cert_issuer",
-			Value: internalIssuer,
-		})
-
-	case acmeIssuer != nil:
-		// fill in global defaults, if configured
-		if email := h.Option("email"); email != nil && acmeIssuer.Email == "" {
-			acmeIssuer.Email = email.(string)
-		}
-		if acmeCA := h.Option("acme_ca"); acmeCA != nil && acmeIssuer.CA == "" {
-			acmeIssuer.CA = acmeCA.(string)
-		}
-		if caPemFile := h.Option("acme_ca_root"); caPemFile != nil {
-			acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, caPemFile.(string))
-		}
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.cert_issuer",
 			Value: disambiguateACMEIssuer(acmeIssuer),
 		})
 	}
+	if internalIssuer != nil {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_issuer",
+			Value: internalIssuer,
+		})
+	}
 
 	// on-demand TLS
 	if onDemand {

caddyconfig/httpcaddyfile/tlsapp.go

@@ -110,47 +110,43 @@ func (st ServerType) buildTLSApp(
 
 			// certificate issuers
 			if issuerVals, ok := sblock.pile["tls.cert_issuer"]; ok {
+				var issuers []certmagic.Issuer
 				for _, issuerVal := range issuerVals {
-					issuer := issuerVal.Value.(certmagic.Issuer)
+					ap.Issuers = append(ap.Issuers, issuerVal.Value.(certmagic.Issuer))
-					if ap == catchAllAP && !reflect.DeepEqual(ap.Issuer, issuer) {
+				}
-						return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuer, issuer)
+				if ap == catchAllAP && !reflect.DeepEqual(ap.Issuers, issuers) {
-					}
+					return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuers, issuers)
-					ap.Issuer = issuer
 				}
 			}
 
 			// custom bind host
 			for _, cfgVal := range sblock.pile["bind"] {
-				// if an issuer was already configured and it is NOT an ACME
+				for _, iss := range ap.Issuers {
-				// issuer, skip, since we intend to adjust only ACME issuers
+					// if an issuer was already configured and it is NOT an ACME issuer,
-				var acmeIssuer *caddytls.ACMEIssuer
+					// skip, since we intend to adjust only ACME issuers; ensure we
-				if ap.Issuer != nil {
+					// include any issuer that embeds/wraps an underlying ACME issuer
-					// ensure we include any issuer that embeds/wraps an underlying ACME issuer
+					var acmeIssuer *caddytls.ACMEIssuer
-					type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }
+					if acmeWrapper, ok := iss.(acmeCapable); ok {
-					if acmeWrapper, ok := ap.Issuer.(acmeCapable); ok {
 						acmeIssuer = acmeWrapper.GetACMEIssuer()
-					} else {
-						break
 					}
-				}
+					if acmeIssuer == nil {
+						continue
+					}
 
-				// proceed to configure the ACME issuer's bind host, without
+					// proceed to configure the ACME issuer's bind host, without
-				// overwriting any existing settings
+					// overwriting any existing settings
-				if acmeIssuer == nil {
+					if acmeIssuer.Challenges == nil {
-					acmeIssuer = new(caddytls.ACMEIssuer)
+						acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
-				}
+					}
-				if acmeIssuer.Challenges == nil {
+					if acmeIssuer.Challenges.BindHost == "" {
-					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+						// only binding to one host is supported
-				}
+						var bindHost string
-				if acmeIssuer.Challenges.BindHost == "" {
+						if bindHosts, ok := cfgVal.Value.([]string); ok && len(bindHosts) > 0 {
-					// only binding to one host is supported
+							bindHost = bindHosts[0]
-					var bindHost string
+						}
-					if bindHosts, ok := cfgVal.Value.([]string); ok && len(bindHosts) > 0 {
+						acmeIssuer.Challenges.BindHost = bindHost
-						bindHost = bindHosts[0]
 					}
-					acmeIssuer.Challenges.BindHost = bindHost
 				}
-				ap.Issuer = acmeIssuer // we'll encode it later
 			}
 
 			// first make sure this block is allowed to create an automation policy;
@@ -188,7 +184,7 @@ func (st ServerType) buildTLSApp(
 			// that the internal names can use the internal issuer and
 			// the other names can use the default/public/ACME issuer
 			var ap2 *caddytls.AutomationPolicy
-			if ap.Issuer == nil {
+			if len(ap.Issuers) == 0 {
 				var internal, external []string
 				for _, s := range ap.Subjects {
 					if !certmagic.SubjectQualifiesForCert(s) {
@@ -212,7 +208,7 @@ func (st ServerType) buildTLSApp(
 					apCopy := *ap
 					ap2 = &apCopy
 					ap2.Subjects = internal
-					ap2.IssuerRaw = caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)
+					ap2.IssuersRaw = []json.RawMessage{caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)}
 				}
 			}
 			if tlsApp.Automation == nil {
@@ -277,7 +273,7 @@ func (st ServerType) buildTLSApp(
 	// get internal certificates by default rather than ACME
 	var al caddytls.AutomateLoader
 	internalAP := &caddytls.AutomationPolicy{
-		IssuerRaw: json.RawMessage(`{"module":"internal"}`),
+		IssuersRaw: []json.RawMessage{json.RawMessage(`{"module":"internal"}`)},
 	}
 	for h := range hostsSharedWithHostlessKey {
 		al = append(al, h)
@@ -295,14 +291,48 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, internalAP)
 	}
 
+	// if there are any global options set for issuers (ACME ones in particular), make sure they
+	// take effect in every automation policy that does not have any issuers
+	if tlsApp.Automation != nil {
+		globalEmail := options["email"]
+		globalACMECA := options["acme_ca"]
+		globalACMECARoot := options["acme_ca_root"]
+		globalACMEDNS := options["acme_dns"]
+		globalACMEEAB := options["acme_eab"]
+		hasGlobalACMEDefaults := globalEmail != nil || globalACMECA != nil || globalACMECARoot != nil || globalACMEDNS != nil || globalACMEEAB != nil
+		if hasGlobalACMEDefaults {
+			for _, ap := range tlsApp.Automation.Policies {
+				if len(ap.Issuers) == 0 {
+					acme, zerosslACME := new(caddytls.ACMEIssuer), new(caddytls.ACMEIssuer)
+					zerossl := &caddytls.ZeroSSLIssuer{ACMEIssuer: zerosslACME}
+					ap.Issuers = []certmagic.Issuer{acme, zerossl} // TODO: keep this in sync with Caddy's other issuer defaults elsewhere, like in caddytls/automation.go (DefaultIssuers).
+
+					// if a non-ZeroSSL endpoint is specified, we assume we can't use the ZeroSSL issuer successfully
+					if globalACMECA != nil && !strings.Contains(globalACMECA.(string), "zerossl") {
+						ap.Issuers = []certmagic.Issuer{acme}
+					}
+				}
+			}
+		}
+	}
+
 	// finalize and verify policies; do cleanup
 	if tlsApp.Automation != nil {
-		// encode any issuer values we created, so they will be rendered in the output
+		for i, ap := range tlsApp.Automation.Policies {
-		for _, ap := range tlsApp.Automation.Policies {
+			// ensure all issuers have global defaults filled in
-			if ap.Issuer != nil && ap.IssuerRaw == nil {
+			for j, issuer := range ap.Issuers {
-				// encode issuer now that it's all set up
+				err := fillInGlobalACMEDefaults(issuer, options)
-				issuerName := ap.Issuer.(caddy.Module).CaddyModule().ID.Name()
+				if err != nil {
-				ap.IssuerRaw = caddyconfig.JSONModuleObject(ap.Issuer, "module", issuerName, &warnings)
+					return nil, warnings, fmt.Errorf("filling in global issuer defaults for AP %d, issuer %d: %v", i, j, err)
+				}
+			}
+
+			// encode all issuer values we created, so they will be rendered in the output
+			if len(ap.Issuers) > 0 && ap.IssuersRaw == nil {
+				for _, iss := range ap.Issuers {
+					issuerName := iss.(caddy.Module).CaddyModule().ID.Name()
+					ap.IssuersRaw = append(ap.IssuersRaw, caddyconfig.JSONModuleObject(iss, "module", issuerName, &warnings))
+				}
 			}
 		}
 
@@ -334,6 +364,51 @@ func (st ServerType) buildTLSApp(
 	return tlsApp, warnings, nil
 }
 
+type acmeCapable interface{ GetACMEIssuer() *caddytls.ACMEIssuer }
+
+func fillInGlobalACMEDefaults(issuer certmagic.Issuer, options map[string]interface{}) error {
+	acmeWrapper, ok := issuer.(acmeCapable)
+	if !ok {
+		return nil
+	}
+	acmeIssuer := acmeWrapper.GetACMEIssuer()
+	if acmeIssuer == nil {
+		return nil
+	}
+
+	globalEmail := options["email"]
+	globalACMECA := options["acme_ca"]
+	globalACMECARoot := options["acme_ca_root"]
+	globalACMEDNS := options["acme_dns"]
+	globalACMEEAB := options["acme_eab"]
+
+	if globalEmail != nil && acmeIssuer.Email == "" {
+		acmeIssuer.Email = globalEmail.(string)
+	}
+	if globalACMECA != nil && acmeIssuer.CA == "" {
+		acmeIssuer.CA = globalACMECA.(string)
+	}
+	if globalACMECARoot != nil && !sliceContains(acmeIssuer.TrustedRootsPEMFiles, globalACMECARoot.(string)) {
+		acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, globalACMECARoot.(string))
+	}
+	if globalACMEDNS != nil && (acmeIssuer.Challenges == nil || acmeIssuer.Challenges.DNS == nil) {
+		provName := globalACMEDNS.(string)
+		dnsProvModule, err := caddy.GetModule("dns.providers." + provName)
+		if err != nil {
+			return fmt.Errorf("getting DNS provider module named '%s': %v", provName, err)
+		}
+		acmeIssuer.Challenges = &caddytls.ChallengesConfig{
+			DNS: &caddytls.DNSChallengeConfig{
+				ProviderRaw: caddyconfig.JSONModuleObject(dnsProvModule.New(), "name", provName, nil),
+			},
+		}
+	}
+	if globalACMEEAB != nil && acmeIssuer.ExternalAccount == nil {
+		acmeIssuer.ExternalAccount = globalACMEEAB.(*acme.EAB)
+	}
+	return nil
+}
+
 // newBaseAutomationPolicy returns a new TLS automation policy that gets
 // its values from the global options map. It should be used as the base
 // for any other automation policies. A nil policy (and no error) will be
@@ -341,17 +416,10 @@ func (st ServerType) buildTLSApp(
 // true, a non-nil value will always be returned (unless there is an error).
 func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddyconfig.Warning, always bool) (*caddytls.AutomationPolicy, error) {
 	issuer, hasIssuer := options["cert_issuer"]
-
+	_, hasLocalCerts := options["local_certs"]
-	acmeCA, hasACMECA := options["acme_ca"]
-	acmeCARoot, hasACMECARoot := options["acme_ca_root"]
-	acmeDNS, hasACMEDNS := options["acme_dns"]
-	acmeEAB, hasACMEEAB := options["acme_eab"]
-
-	email, hasEmail := options["email"]
-	localCerts, hasLocalCerts := options["local_certs"]
 	keyType, hasKeyType := options["key_type"]
 
-	hasGlobalAutomationOpts := hasIssuer || hasACMECA || hasACMECARoot || hasACMEDNS || hasACMEEAB || hasEmail || hasLocalCerts || hasKeyType
+	hasGlobalAutomationOpts := hasIssuer || hasLocalCerts || hasKeyType
 
 	// if there are no global options related to automation policies
 	// set, then we can just return right away
@@ -363,48 +431,18 @@ func newBaseAutomationPolicy(options map[string]interface{}, warnings []caddycon
 	}
 
 	ap := new(caddytls.AutomationPolicy)
-	if keyType != nil {
+	if hasKeyType {
 		ap.KeyType = keyType.(string)
 	}
 
+	if hasIssuer && hasLocalCerts {
+		return nil, fmt.Errorf("global options are ambiguous: local_certs is confusing when combined with cert_issuer, because local_certs is also a specific kind of issuer")
+	}
+
 	if hasIssuer {
-		if hasACMECA || hasACMEDNS || hasACMEEAB || hasEmail || hasLocalCerts {
+		ap.Issuers = []certmagic.Issuer{issuer.(certmagic.Issuer)}
-			return nil, fmt.Errorf("global options are ambiguous: cert_issuer is confusing when combined with acme_*, email, or local_certs options")
+	} else if hasLocalCerts {
-		}
+		ap.Issuers = []certmagic.Issuer{new(caddytls.InternalIssuer)}
-		ap.Issuer = issuer.(certmagic.Issuer)
-	} else if localCerts != nil {
-		// internal issuer enabled trumps any ACME configurations; useful in testing
-		ap.Issuer = new(caddytls.InternalIssuer) // we'll encode it later
-	} else {
-		if acmeCA == nil {
-			acmeCA = ""
-		}
-		if email == nil {
-			email = ""
-		}
-		mgr := &caddytls.ACMEIssuer{
-			CA:    acmeCA.(string),
-			Email: email.(string),
-		}
-		if acmeDNS != nil {
-			provName := acmeDNS.(string)
-			dnsProvModule, err := caddy.GetModule("dns.providers." + provName)
-			if err != nil {
-				return nil, fmt.Errorf("getting DNS provider module named '%s': %v", provName, err)
-			}
-			mgr.Challenges = &caddytls.ChallengesConfig{
-				DNS: &caddytls.DNSChallengeConfig{
-					ProviderRaw: caddyconfig.JSONModuleObject(dnsProvModule.New(), "name", provName, &warnings),
-				},
-			}
-		}
-		if acmeCARoot != nil {
-			mgr.TrustedRootsPEMFiles = []string{acmeCARoot.(string)}
-		}
-		if acmeEAB != nil {
-			mgr.ExternalAccount = acmeEAB.(*acme.EAB)
-		}
-		ap.Issuer = disambiguateACMEIssuer(mgr) // we'll encode it later
 	}
 
 	return ap, nil
@@ -463,7 +501,7 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
 			// otherwise the one without any subjects (a catch-all) would be
 			// eaten up by the one with subjects; and if both have subjects, we
 			// need to combine their lists
-			if bytes.Equal(aps[i].IssuerRaw, aps[j].IssuerRaw) &&
+			if reflect.DeepEqual(aps[i].IssuersRaw, aps[j].IssuersRaw) &&
 				bytes.Equal(aps[i].StorageRaw, aps[j].StorageRaw) &&
 				aps[i].MustStaple == aps[j].MustStaple &&
 				aps[i].KeyType == aps[j].KeyType &&

caddytest/integration/caddyfile_adapt/global_options.txt

@@ -54,9 +54,11 @@
 			"automation": {
 				"policies": [
 					{
-						"issuer": {
+						"issuers": [
-							"module": "internal"
+							{
-						},
+								"module": "internal"
+							}
+						],
 						"key_type": "ed25519"
 					}
 				],

caddytest/integration/caddyfile_adapt/global_options_acme.txt

@@ -57,18 +57,20 @@
 			"automation": {
 				"policies": [
 					{
-						"issuer": {
+						"issuers": [
-							"ca": "https://example.com",
+							{
-							"email": "test@example.com",
+								"ca": "https://example.com",
-							"external_account": {
+								"email": "test@example.com",
-								"key_id": "4K2scIVbBpNd-78scadB2g",
+								"external_account": {
-								"mac_key": "abcdefghijklmnopqrstuvwx-abcdefghijklnopqrstuvwxyz12ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh"
+									"key_id": "4K2scIVbBpNd-78scadB2g",
-							},
+									"mac_key": "abcdefghijklmnopqrstuvwx-abcdefghijklnopqrstuvwxyz12ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgh"
-							"module": "acme",
+								},
-							"trusted_roots_pem_files": [
+								"module": "acme",
-								"/path/to/ca.crt"
+								"trusted_roots_pem_files": [
-							]
+									"/path/to/ca.crt"
-						},
+								]
+							}
+						],
 						"key_type": "ed25519"
 					}
 				],

caddytest/integration/caddyfile_adapt/global_options_admin.txt

@@ -62,9 +62,11 @@
 			"automation": {
 				"policies": [
 					{
-						"issuer": {
+						"issuers": [
-							"module": "internal"
+							{
-						},
+								"module": "internal"
+							}
+						],
 						"key_type": "ed25519"
 					}
 				],

caddytest/integration/caddyfile_adapt/tls_automation_policies.txt

@@ -54,24 +54,30 @@ foo.tld, www.foo.tld {
 							"foo.tld",
 							"www.foo.tld"
 						],
-						"issuer": {
+						"issuers": [
-							"module": "internal"
+							{
-						}
+								"module": "internal"
+							}
+						]
 					},
 					{
 						"subjects": [
 							"*.*.tld",
 							"*.tld"
 						],
-						"issuer": {
+						"issuers": [
-							"module": "internal"
+							{
-						},
+								"module": "internal"
+							}
+						],
 						"on_demand": true
 					},
 					{
-						"issuer": {
+						"issuers": [
-							"module": "internal"
+							{
-						}
+								"module": "internal"
+							}
+						]
 					}
 				]
 			}

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.1.0
 	github.com/alecthomas/chroma v0.8.0
 	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
-	github.com/caddyserver/certmagic v0.12.1-0.20201112220015-12df2be5a021
+	github.com/caddyserver/certmagic v0.12.1-0.20201116175341-0f8a9f688760
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/cel-go v0.5.1

go.sum

@@ -85,8 +85,8 @@ github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTK
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/caddyserver/certmagic v0.12.1-0.20201112220015-12df2be5a021 h1:VY4JahIicW/GT84KoIm+TSKwMHj08NyPJhsGypTWJn4=
+github.com/caddyserver/certmagic v0.12.1-0.20201116175341-0f8a9f688760 h1:h7KGtOb9TAfZp2/KwPd9iyqiLVZMWbpx5Mu0Her2iRw=
-github.com/caddyserver/certmagic v0.12.1-0.20201112220015-12df2be5a021/go.mod h1:tr26xh+9fY5dN0J6IPAlMj07qpog22PJKa7Nw7j835U=
+github.com/caddyserver/certmagic v0.12.1-0.20201116175341-0f8a9f688760/go.mod h1:tr26xh+9fY5dN0J6IPAlMj07qpog22PJKa7Nw7j835U=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

modules/caddyhttp/autohttps.go

@@ -241,7 +241,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 	// we now have a list of all the unique names for which we need certs;
 	// turn the set into a slice so that phase 2 can use it
 	app.allCertDomains = make([]string, 0, len(uniqueDomainsForCerts))
-	var internal, external []string
+	var internal []string
 uniqueDomainsLoop:
 	for d := range uniqueDomainsForCerts {
 		// whether or not there is already an automation policy for this
@@ -264,15 +264,13 @@ uniqueDomainsLoop:
 
 		// if no automation policy exists for the name yet, we
 		// will associate it with an implicit one
-		if certmagic.SubjectQualifiesForPublicCert(d) {
+		if !certmagic.SubjectQualifiesForPublicCert(d) {
-			external = append(external, d)
-		} else {
 			internal = append(internal, d)
 		}
 	}
 
 	// ensure there is an automation policy to handle these certs
-	err := app.createAutomationPolicies(ctx, external, internal)
+	err := app.createAutomationPolicies(ctx, internal)
 	if err != nil {
 		return err
 	}
@@ -430,7 +428,7 @@ redirServersLoop:
 // automation policy exists, it will be shallow-copied and used as the
 // base for the new ones (this is important for preserving behavior the
 // user intends to be "defaults").
-func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, internalNames []string) error {
+func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []string) error {
 	// before we begin, loop through the existing automation policies
 	// and, for any ACMEIssuers we find, make sure they're filled in
 	// with default values that might be specified in our HTTP app; also
@@ -447,16 +445,23 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 		// set up default issuer -- honestly, this is only
 		// really necessary because the HTTP app is opinionated
 		// and has settings which could be inferred as new
-		// defaults for the ACMEIssuer in the TLS app
+		// defaults for the ACMEIssuer in the TLS app (such as
-		if ap.Issuer == nil {
+		// what the HTTP and HTTPS ports are)
-			ap.Issuer = new(caddytls.ACMEIssuer)
+		if ap.Issuers == nil {
-		}
+			var err error
-		if acmeIssuer, ok := ap.Issuer.(acmeCapable); ok {
+			ap.Issuers, err = caddytls.DefaultIssuers(ctx)
-			err := app.fillInACMEIssuer(acmeIssuer.GetACMEIssuer())
 			if err != nil {
 				return err
 			}
 		}
+		for _, iss := range ap.Issuers {
+			if acmeIssuer, ok := iss.(acmeCapable); ok {
+				err := app.fillInACMEIssuer(acmeIssuer.GetACMEIssuer())
+				if err != nil {
+					return err
+				}
+			}
+		}
 
 		// while we're here, is this the catch-all/base policy?
 		if !foundBasePolicy && len(ap.Subjects) == 0 {
@@ -471,11 +476,14 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 	}
 
 	// if the basePolicy has an existing ACMEIssuer (particularly to
-	// include any type that embeds/wraps an ACMEIssuer), let's use it,
+	// include any type that embeds/wraps an ACMEIssuer), let's use it
-	// otherwise we'll make one
+	// (I guess we just use the first one?), otherwise we'll make one
 	var baseACMEIssuer *caddytls.ACMEIssuer
-	if acmeWrapper, ok := basePolicy.Issuer.(acmeCapable); ok {
+	for _, iss := range basePolicy.Issuers {
-		baseACMEIssuer = acmeWrapper.GetACMEIssuer()
+		if acmeWrapper, ok := iss.(acmeCapable); ok {
+			baseACMEIssuer = acmeWrapper.GetACMEIssuer()
+			break
+		}
 	}
 	if baseACMEIssuer == nil {
 		// note that this happens if basePolicy.Issuer is nil
@@ -485,7 +493,7 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 
 	// if there was a base policy to begin with, we already
 	// filled in its issuer's defaults; if there wasn't, we
-	// stil need to do that
+	// still need to do that
 	if !foundBasePolicy {
 		err := app.fillInACMEIssuer(baseACMEIssuer)
 		if err != nil {
@@ -494,8 +502,20 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 	}
 
 	// never overwrite any other issuer that might already be configured
-	if basePolicy.Issuer == nil {
+	if basePolicy.Issuers == nil {
-		basePolicy.Issuer = baseACMEIssuer
+		var err error
+		basePolicy.Issuers, err = caddytls.DefaultIssuers(ctx)
+		if err != nil {
+			return err
+		}
+		for _, iss := range basePolicy.Issuers {
+			if acmeIssuer, ok := iss.(acmeCapable); ok {
+				err := app.fillInACMEIssuer(acmeIssuer.GetACMEIssuer())
+				if err != nil {
+					return err
+				}
+			}
+		}
 	}
 
 	if !foundBasePolicy {
@@ -549,8 +569,7 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, interna
 		// of names that would normally use the production API;
 		// anyway, that gets into the weeds a bit...
 		newPolicy.Subjects = internalNames
-		newPolicy.Issuer = internalIssuer
+		newPolicy.Issuers = []certmagic.Issuer{internalIssuer}
-
 		err := app.tlsApp.AddAutomationPolicy(newPolicy)
 		if err != nil {
 			return err

modules/caddytls/acmeissuer.go

@@ -97,6 +97,15 @@ func (ACMEIssuer) CaddyModule() caddy.ModuleInfo {
 func (iss *ACMEIssuer) Provision(ctx caddy.Context) error {
 	iss.logger = ctx.Logger(iss)
 
+	// expand email address, if non-empty
+	if iss.Email != "" {
+		email, err := caddy.NewReplacer().ReplaceOrErr(iss.Email, true, true)
+		if err != nil {
+			return fmt.Errorf("expanding email address '%s': %v", iss.Email, err)
+		}
+		iss.Email = email
+	}
+
 	// DNS providers
 	if iss.Challenges != nil && iss.Challenges.DNS != nil && iss.Challenges.DNS.ProviderRaw != nil {
 		val, err := ctx.LoadModule(iss.Challenges.DNS, "ProviderRaw")

modules/caddytls/automation.go

@@ -23,7 +23,6 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/certmagic"
 	"github.com/mholt/acmez"
-	"go.uber.org/zap"
 )
 
 // AutomationConfig governs the automated management of TLS certificates.
@@ -72,8 +71,13 @@ type AutomationPolicy struct {
 	// Which subjects (hostnames or IP addresses) this policy applies to.
 	Subjects []string `json:"subjects,omitempty"`
 
-	// The module that will issue certificates. Default: internal if all
+	// The modules that may issue certificates. Default: internal if all
-	// subjects do not qualify for public certificates; othewise acme.
+	// subjects do not qualify for public certificates; othewise acme and
+	// zerossl.
+	IssuersRaw []json.RawMessage `json:"issuers,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
+
+	// DEPRECATED: Use `issuers` instead (November 2020). This field will
+	// be removed in the future.
 	IssuerRaw json.RawMessage `json:"issuer,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
 
 	// If true, certificates will be requested with MustStaple. Not all
@@ -103,10 +107,10 @@ type AutomationPolicy struct {
 	// load.
 	OnDemand bool `json:"on_demand,omitempty"`
 
-	// Issuer stores the decoded issuer parameters. This is only
+	// Issuers stores the decoded issuer parameters. This is only
-	// used to populate an underlying certmagic.Config's Issuer
+	// used to populate an underlying certmagic.Config's Issuers
 	// field; it is not referenced thereafter.
-	Issuer certmagic.Issuer `json:"-"`
+	Issuers []certmagic.Issuer `json:"-"`
 
 	magic   *certmagic.Config
 	storage certmagic.Storage
@@ -150,34 +154,30 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 		}
 	}
 
-	// if this automation policy has no Issuer defined, and
+	// TODO: IssuerRaw field deprecated as of November 2020 - remove this shim after deprecation is complete
-	// none of the subjects qualify for a public certificate,
+	if ap.IssuerRaw != nil {
-	// set the issuer to internal so that these names can all
+		tlsApp.logger.Warn("the 'issuer' field is deprecated and will be removed in the future; use 'issuers' instead; your issuer has been appended automatically for now")
-	// get certificates; critically, we can only do this if an
+		ap.IssuersRaw = append(ap.IssuersRaw, ap.IssuerRaw)
-	// issuer is not explicitly configured (IssuerRaw, vs. just
-	// Issuer) AND if the list of subjects is non-empty
-	if ap.IssuerRaw == nil && len(ap.Subjects) > 0 {
-		var anyPublic bool
-		for _, s := range ap.Subjects {
-			if certmagic.SubjectQualifiesForPublicCert(s) {
-				anyPublic = true
-				break
-			}
-		}
-		if !anyPublic {
-			tlsApp.logger.Info("setting internal issuer for automation policy that has only internal subjects but no issuer configured",
-				zap.Strings("subjects", ap.Subjects))
-			ap.IssuerRaw = json.RawMessage(`{"module":"internal"}`)
-		}
 	}
 
-	// load and provision any explicitly-configured issuer module
+	// load and provision any explicitly-configured issuer modules
-	if ap.IssuerRaw != nil {
+	if ap.IssuersRaw != nil {
-		val, err := tlsApp.ctx.LoadModule(ap, "IssuerRaw")
+		val, err := tlsApp.ctx.LoadModule(ap, "IssuersRaw")
 		if err != nil {
 			return fmt.Errorf("loading TLS automation management module: %s", err)
 		}
-		ap.Issuer = val.(certmagic.Issuer)
+		for _, issVal := range val.([]interface{}) {
+			ap.Issuers = append(ap.Issuers, issVal.(certmagic.Issuer))
+		}
+	}
+
+	issuers := ap.Issuers
+	if len(issuers) == 0 {
+		var err error
+		issuers, err = DefaultIssuers(tlsApp.ctx)
+		if err != nil {
+			return err
+		}
 	}
 
 	keyType := ap.KeyType
@@ -206,12 +206,9 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 		KeySource:          keySource,
 		OnDemand:           ond,
 		Storage:            storage,
-		Issuer:             ap.Issuer, // if nil, certmagic.New() will create one
+		Issuers:            issuers,
 		Logger:             tlsApp.logger,
 	}
-	if rev, ok := ap.Issuer.(certmagic.Revoker); ok {
-		template.Revoker = rev
-	}
 	ap.magic = certmagic.New(tlsApp.certCache, template)
 
 	// sometimes issuers may need the parent certmagic.Config in
@@ -219,13 +216,32 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 	// access to the correct storage and cache so it can solve
 	// ACME challenges -- it's an annoying, inelegant circular
 	// dependency that I don't know how to resolve nicely!)
-	if annoying, ok := ap.Issuer.(ConfigSetter); ok {
+	for _, issuer := range ap.magic.Issuers {
-		annoying.SetConfig(ap.magic)
+		if annoying, ok := issuer.(ConfigSetter); ok {
+			annoying.SetConfig(ap.magic)
+		}
 	}
 
 	return nil
 }
 
+// DefaultIssuers returns empty but provisioned default Issuers.
+// This function is experimental and has no compatibility promises.
+func DefaultIssuers(ctx caddy.Context) ([]certmagic.Issuer, error) {
+	acme := new(ACMEIssuer)
+	err := acme.Provision(ctx)
+	if err != nil {
+		return nil, err
+	}
+	zerossl := new(ZeroSSLIssuer)
+	err = zerossl.Provision(ctx)
+	if err != nil {
+		return nil, err
+	}
+	// TODO: eventually, insert ZeroSSL into first position in the slice -- see also httpcaddyfile/tlsapp.go for where similar defaults are configured
+	return []certmagic.Issuer{acme, zerossl}, nil
+}
+
 // ChallengesConfig configures the ACME challenges.
 type ChallengesConfig struct {
 	// HTTP configures the ACME HTTP challenge. This

modules/caddytls/tls.go

@@ -137,7 +137,7 @@ func (t *TLS) Provision(ctx caddy.Context) error {
 			continue
 		}
 		t.Automation.defaultInternalAutomationPolicy = &AutomationPolicy{
-			IssuerRaw: json.RawMessage(`{"module":"internal"}`),
+			IssuersRaw: []json.RawMessage{json.RawMessage(`{"module":"internal"}`)},
 		}
 		err = t.Automation.defaultInternalAutomationPolicy.Provision(t)
 		if err != nil {
@@ -303,20 +303,22 @@ func (t *TLS) Manage(names []string) error {
 
 // HandleHTTPChallenge ensures that the HTTP challenge is handled for the
 // certificate named by r.Host, if it is an HTTP challenge request. It
-// requires that the automation policy for r.Host has an issue of type
+// requires that the automation policy for r.Host has an issuer of type
-// *certmagic.ACMEManager.
+// *certmagic.ACMEManager, or one that is ACME-enabled (GetACMEIssuer()).
 func (t *TLS) HandleHTTPChallenge(w http.ResponseWriter, r *http.Request) bool {
 	if !certmagic.LooksLikeHTTPChallenge(r) {
 		return false
 	}
+	// try all the issuers until we find the one that initiated the challenge
 	ap := t.getAutomationPolicyForName(r.Host)
-	if ap.magic.Issuer == nil {
-		return false
-	}
 	type acmeCapable interface{ GetACMEIssuer() *ACMEIssuer }
-	if am, ok := ap.magic.Issuer.(acmeCapable); ok {
+	for _, iss := range ap.magic.Issuers {
-		iss := am.GetACMEIssuer()
+		if am, ok := iss.(acmeCapable); ok {
-		return certmagic.NewACMEManager(iss.magic, iss.template).HandleHTTPChallenge(w, r)
+			iss := am.GetACMEIssuer()
+			if certmagic.NewACMEManager(iss.magic, iss.template).HandleHTTPChallenge(w, r) {
+				return true
+			}
+		}
 	}
 	return false
 }

modules/caddytls/zerosslissuer.go

@@ -59,16 +59,13 @@ func (*ZeroSSLIssuer) CaddyModule() caddy.ModuleInfo {
 // Provision sets up iss.
 func (iss *ZeroSSLIssuer) Provision(ctx caddy.Context) error {
 	iss.logger = ctx.Logger(iss)
-
 	if iss.ACMEIssuer == nil {
 		iss.ACMEIssuer = new(ACMEIssuer)
 	}
-	err := iss.ACMEIssuer.Provision(ctx)
+	if iss.ACMEIssuer.CA == "" {
-	if err != nil {
+		iss.ACMEIssuer.CA = certmagic.ZeroSSLProductionCA
-		return err
 	}
-
+	return iss.ACMEIssuer.Provision(ctx)
-	return nil
 }
 
 func (iss *ZeroSSLIssuer) newAccountCallback(ctx context.Context, am *certmagic.ACMEManager, _ acme.Account) error {
@@ -86,26 +83,22 @@ func (iss *ZeroSSLIssuer) generateEABCredentials(ctx context.Context) (*acme.EAB
 
 	// there are two ways to generate EAB credentials: authenticated with
 	// their API key, or unauthenticated with their email address
-	switch {
+	if iss.APIKey != "" {
-	case iss.APIKey != "":
 		apiKey := caddy.NewReplacer().ReplaceAll(iss.APIKey, "")
 		if apiKey == "" {
 			return nil, fmt.Errorf("missing API key: '%v'", iss.APIKey)
 		}
 		qs := url.Values{"access_key": []string{apiKey}}
 		endpoint = fmt.Sprintf("%s/eab-credentials?%s", zerosslAPIBase, qs.Encode())
-
+	} else {
-	case iss.Email != "":
+		email := iss.Email
-		email := caddy.NewReplacer().ReplaceAll(iss.Email, "")
 		if email == "" {
-			return nil, fmt.Errorf("missing email: '%v'", iss.Email)
+			iss.logger.Warn("missing email address for ZeroSSL; it is strongly recommended to set one for next time")
+			email = "caddy@zerossl.com" // special email address that preserves backwards-compat, but which black-holes dashboard features, oh well
 		}
 		endpoint = zerosslAPIBase + "/eab-credentials-email"
 		form := url.Values{"email": []string{email}}
 		body = strings.NewReader(form.Encode())
-
-	default:
-		return nil, fmt.Errorf("must configure either an API key or email address to use ZeroSSL without explicit EAB")
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, body)
@@ -161,9 +154,6 @@ func (iss *ZeroSSLIssuer) generateEABCredentials(ctx context.Context) (*acme.EAB
 func (iss *ZeroSSLIssuer) initialize() {
 	iss.mu.Lock()
 	defer iss.mu.Unlock()
-	if iss.template.CA == "" {
-		iss.template.CA = zerosslACMEDirectory
-	}
 	if iss.template.NewAccountFunc == nil {
 		iss.template.NewAccountFunc = iss.newAccountCallback
 	}
@@ -195,15 +185,18 @@ func (iss *ZeroSSLIssuer) Revoke(ctx context.Context, cert certmagic.Certificate
 
 // UnmarshalCaddyfile deserializes Caddyfile tokens into iss.
 //
-//     ... zerossl <api_key> {
+//     ... zerossl [<api_key>] {
 //         ...
 //     }
 //
 // Any of the subdirectives for the ACME issuer can be used in the block.
 func (iss *ZeroSSLIssuer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	for d.Next() {
-		if !d.AllArgs(&iss.APIKey) {
+		if d.NextArg() {
-			return d.ArgErr()
+			iss.APIKey = d.Val()
+			if d.NextArg() {
+				return d.ArgErr()
+			}
 		}
 
 		if iss.ACMEIssuer == nil {
@@ -217,10 +210,7 @@ func (iss *ZeroSSLIssuer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	return nil
 }
 
-const (
+const zerosslAPIBase = "https://api.zerossl.com/acme"
-	zerosslACMEDirectory = "https://acme.zerossl.com/v2/DV90"
-	zerosslAPIBase       = "https://api.zerossl.com/acme"
-)
 
 // Interface guards
 var (