From 82cbd7a96b308a2fb711c9bed32668a034a8380c Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Fri, 17 Feb 2017 14:07:57 -0700 Subject: [PATCH] Detect HTTPS interception (#1430) * WIP: Implement HTTPS interception detection by Durumeric, et. al. Special thanks to @FiloSottile for guidance with the custom listener. * Add {{.IsMITM}} context action and {mitm} placeholder * Improve MITM detection heuristics for Firefox and Edge * Add tests for MITM detection heuristics * Improve Safari heuristics for interception detection * Read ClientHello during first Read() instead of during Accept() As far as I can tell, reading the ClientHello during Accept() prevents new connections from being accepted during the read. Since Read() should be called in its own goroutine, this keeps Accept() non-blocking. * Clean up MITM detection handler; make possible to close connection * Use standard lib cipher suite values when possible * Improve Edge heuristics and test cases * Refactor MITM checking logic; add some debug statements for now * Fix bug in MITM heuristic tests and actual heuristic code * Fix gofmt * Remove debug statements; preparing for merge --- CONTRIBUTING.md | 4 +- caddy.go | 2 + caddyhttp/httpserver/context.go | 11 + caddyhttp/httpserver/mitm.go | 560 ++++++++++++++++++++++++++++++ caddyhttp/httpserver/mitm_test.go | 206 +++++++++++ caddyhttp/httpserver/replacer.go | 9 + caddyhttp/httpserver/server.go | 21 +- 7 files changed, 811 insertions(+), 2 deletions(-) create mode 100644 caddyhttp/httpserver/mitm.go create mode 100644 caddyhttp/httpserver/mitm_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb916c55..43a2ae8e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,9 @@ one collaborator who did not open the pull request before merging. This will help ensure high code quality as new collaborators are added to the project. Read [CodeReviewComments](https://github.com/golang/go/wiki/CodeReviewComments) -on the Go wiki for an idea of what we look for in good, clean Go code. +on the Go wiki for an idea of what we look for in good, clean Go code, and +check out [what Linus suggests](https://gist.github.com/matthewhudson/1475276) +for good commit messages. diff --git a/caddy.go b/caddy.go index b55c302f..0df59fb7 100644 --- a/caddy.go +++ b/caddy.go @@ -5,6 +5,8 @@ // 1. Set the AppName and AppVersion variables. // 2. Call LoadCaddyfile() to get the Caddyfile. // Pass in the name of the server type (like "http"). +// Make sure the server type's package is imported +// (import _ "github.com/mholt/caddy/caddyhttp"). // 3. Call caddy.Start() to start Caddy. You get back // an Instance, on which you can call Restart() to // restart it or Stop() to stop it. diff --git a/caddyhttp/httpserver/context.go b/caddyhttp/httpserver/context.go index b7d1065c..549ced88 100644 --- a/caddyhttp/httpserver/context.go +++ b/caddyhttp/httpserver/context.go @@ -321,3 +321,14 @@ func (c Context) Files(name string) ([]string, error) { return names, nil } + +// IsMITM returns true if it seems likely that the TLS connection +// is being intercepted. +func (c Context) IsMITM() bool { + if val, ok := c.Req.Context().Value(CtxKey("mitm")).(bool); ok { + return val + } + return false +} + +type CtxKey string diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go new file mode 100644 index 00000000..acb23244 --- /dev/null +++ b/caddyhttp/httpserver/mitm.go @@ -0,0 +1,560 @@ +package httpserver + +import ( + "bytes" + "context" + "crypto/tls" + "io" + "net" + "net/http" + "strings" + "sync" + "time" +) + +// tlsHandler is a http.Handler that will inject a value +// into the request context indicating if the TLS +// connection is likely being intercepted. +type tlsHandler struct { + next http.Handler + listener *tlsHelloListener + closeOnMITM bool // whether to close connection on MITM; TODO: expose through new directive +} + +// ServeHTTP checks the User-Agent. For the four main browsers (Chrome, +// Edge, Firefox, and Safari) indicated by the User-Agent, the properties +// of the TLS Client Hello will be compared. The context value "mitm" will +// be set to a value indicating if it is likely that the underlying TLS +// connection is being intercepted. +// +// Note that due to Microsoft's decision to intentionally make IE/Edge +// user agents obscure (and look like other browsers), this may offer +// less accuracy for IE/Edge clients. +// +// This MITM detection capability is based on research done by Durumeric, +// Halderman, et. al. in "The Security Impact of HTTPS Interception" (NDSS '17): +// https://jhalderm.com/pub/papers/interception-ndss17.pdf +func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.listener.helloInfosMu.RLock() + info := h.listener.helloInfos[r.RemoteAddr] + h.listener.helloInfosMu.RUnlock() + + ua := r.Header.Get("User-Agent") + + var checked, mitm bool + if r.Header.Get("X-BlueCoat-Via") != "" || // Blue Coat (masks User-Agent header to generic values) + r.Header.Get("X-FCCKV2") != "" || // Fortinet + info.advertisesHeartbeatSupport() { // no major browsers have ever implemented Heartbeat + checked = true + mitm = true + } else if strings.Contains(ua, "Edge") || strings.Contains(ua, "MSIE") || + strings.Contains(ua, "Trident") { + checked = true + mitm = !info.looksLikeEdge() + } else if strings.Contains(ua, "Chrome") { + checked = true + mitm = !info.looksLikeChrome() + } else if strings.Contains(ua, "Firefox") { + checked = true + mitm = !info.looksLikeFirefox() + } else if strings.Contains(ua, "Safari") { + checked = true + mitm = !info.looksLikeSafari() + } + + if checked { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), mitm)) + } + + if mitm && h.closeOnMITM { + // TODO: This termination might need to happen later in the middleware + // chain in order to be picked up by the log directive, in case the site + // owner still wants to log this event. It'll probably require a new + // directive. If this feature is useful, we can finish implementing this. + r.Close = true + return + } + + h.next.ServeHTTP(w, r) +} + +type clientHelloConn struct { + net.Conn + readHello bool + listener *tlsHelloListener +} + +func (c *clientHelloConn) Read(b []byte) (n int, err error) { + if !c.readHello { + // Read the header bytes. + hdr := make([]byte, 5) + n, err := io.ReadFull(c.Conn, hdr) + if err != nil { + return n, err + } + + // Get the length of the ClientHello message and read it as well. + length := uint16(hdr[3])<<8 | uint16(hdr[4]) + hello := make([]byte, int(length)) + n, err = io.ReadFull(c.Conn, hello) + if err != nil { + return n, err + } + + // Parse the ClientHello and store it in the map. + rawParsed := parseRawClientHello(hello) + c.listener.helloInfosMu.Lock() + c.listener.helloInfos[c.Conn.RemoteAddr().String()] = rawParsed + c.listener.helloInfosMu.Unlock() + + // Since we buffered the header and ClientHello, pretend we were + // never here by lining up the buffered values to be read with a + // custom connection type, followed by the rest of the actual + // underlying connection. + mr := io.MultiReader(bytes.NewReader(hdr), bytes.NewReader(hello), c.Conn) + mc := multiConn{Conn: c.Conn, reader: mr} + + c.Conn = mc + + c.readHello = true + } + return c.Conn.Read(b) +} + +// multiConn is a net.Conn that reads from the +// given reader instead of the wire directly. This +// is useful when some of the connection has already +// been read (like the TLS Client Hello) and the +// reader is a io.MultiReader that starts with +// the contents of the buffer. +type multiConn struct { + net.Conn + reader io.Reader +} + +// Read reads from mc.reader. +func (mc multiConn) Read(b []byte) (n int, err error) { + return mc.reader.Read(b) +} + +// parseRawClientHello parses data which contains the raw +// TLS Client Hello message. It extracts relevant information +// into info. Any error reading the Client Hello (such as +// insufficient length or invalid length values) results in +// a silent error and an incomplete info struct, since there +// is no good way to handle an error like this during Accept(). +// The data is expected to contain the whole ClientHello and +// ONLY the ClientHello. +// +// The majority of this code is borrowed from the Go standard +// library, which is (c) The Go Authors. It has been modified +// to fit this use case. +func parseRawClientHello(data []byte) (info rawHelloInfo) { + if len(data) < 42 { + return + } + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return + } + data = data[39+sessionIdLen:] + if len(data) < 2 { + return + } + // cipherSuiteLen is the number of bytes of cipher suite numbers. Since + // they are uint16s, the number must be even. + cipherSuiteLen := int(data[0])<<8 | int(data[1]) + if cipherSuiteLen%2 == 1 || len(data) < 2+cipherSuiteLen { + return + } + numCipherSuites := cipherSuiteLen / 2 + // read in the cipher suites + info.cipherSuites = make([]uint16, numCipherSuites) + for i := 0; i < numCipherSuites; i++ { + info.cipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i]) + } + data = data[2+cipherSuiteLen:] + if len(data) < 1 { + return + } + // read in the compression methods + compressionMethodsLen := int(data[0]) + if len(data) < 1+compressionMethodsLen { + return + } + info.compressionMethods = data[1 : 1+compressionMethodsLen] + + data = data[1+compressionMethodsLen:] + + // ClientHello is optionally followed by extension data + if len(data) < 2 { + return + } + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if extensionsLength != len(data) { + return + } + + // read in each extension, and extract any relevant information + // from extensions we care about + for len(data) != 0 { + if len(data) < 4 { + return + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return + } + + // record that the client advertised support for this extension + info.extensions = append(info.extensions, extension) + + switch extension { + case extensionSupportedCurves: + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + if length < 2 { + return + } + l := int(data[0])<<8 | int(data[1]) + if l%2 == 1 || length != l+2 { + return + } + numCurves := l / 2 + info.curves = make([]tls.CurveID, numCurves) + d := data[2:] + for i := 0; i < numCurves; i++ { + info.curves[i] = tls.CurveID(d[0])<<8 | tls.CurveID(d[1]) + d = d[2:] + } + case extensionSupportedPoints: + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + if length < 1 { + return + } + l := int(data[0]) + if length != l+1 { + return + } + info.points = make([]uint8, l) + copy(info.points, data[1:]) + } + + data = data[length:] + } + + return +} + +// newTLSListener returns a new tlsHelloListener that wraps ln. +func newTLSListener(ln net.Listener, config *tls.Config, readTimeout time.Duration) *tlsHelloListener { + return &tlsHelloListener{ + Listener: ln, + config: config, + readTimeout: readTimeout, + helloInfos: make(map[string]rawHelloInfo), + } +} + +// tlsHelloListener is a TLS listener that is specially designed +// to read the ClientHello manually so we can extract necessary +// information from it. Each ClientHello message is mapped by +// the remote address of the client, which must be removed when +// the connection is closed (use ConnState). +type tlsHelloListener struct { + net.Listener + config *tls.Config + readTimeout time.Duration + helloInfos map[string]rawHelloInfo + helloInfosMu sync.RWMutex +} + +// Accept waits for and returns the next connection to the listener. +// After it accepts the underlying connection, it reads the +// ClientHello message and stores the parsed data into a map on l. +func (l *tlsHelloListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + helloConn := &clientHelloConn{Conn: conn, listener: l} + return tls.Server(helloConn, l.config), nil +} + +// rawHelloInfo contains the "raw" data parsed from the TLS +// Client Hello. No interpretation is done on the raw data. +// +// The methods on this type implement heuristics described +// by Durumeric, Halderman, et. al. in +// "The Security Impact of HTTPS Interception": +// https://jhalderm.com/pub/papers/interception-ndss17.pdf +type rawHelloInfo struct { + cipherSuites []uint16 + extensions []uint16 + compressionMethods []byte + curves []tls.CurveID + points []uint8 +} + +// advertisesHeartbeatSupport returns true if info indicates +// that the client supports the Heartbeat extension. +func (info rawHelloInfo) advertisesHeartbeatSupport() bool { + for _, ext := range info.extensions { + if ext == extensionHeartbeat { + return true + } + } + return false +} + +// looksLikeFirefox returns true if info looks like a handshake +// from a modern version of Firefox. +func (info rawHelloInfo) looksLikeFirefox() bool { + // "To determine whether a Firefox session has been + // intercepted, we check for the presence and order + // of extensions, cipher suites, elliptic curves, + // EC point formats, and handshake compression methods." + + // We check for the presence and order of the extensions. + // Note: Sometimes padding (21) is present, sometimes not. + // Note: Firefox 51+ does not advertise 0x3374 (13172, NPN). + // Note: Firefox doesn't advertise 0x0 (0, SNI) when connecting to IP addresses. + requiredExtensionsOrder := []uint16{23, 65281, 10, 11, 35, 16, 5, 65283, 13} + if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) { + return false + } + + // We check for both presence of curves and their ordering. + expectedCurves := []tls.CurveID{29, 23, 24, 25} + if len(info.curves) != len(expectedCurves) { + return false + } + for i := range expectedCurves { + if info.curves[i] != expectedCurves[i] { + return false + } + } + + // We check for order of cipher suites but not presence, since + // according to the paper, cipher suites may be not be added + // or reordered by the user, but they may be disabled. + expectedCipherSuiteOrder := []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, // 0xcca9 + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, // 0xcca8 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // 0xc00a + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA, // 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA, // 0x39 + tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f + tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // 0xa + } + return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false) +} + +// looksLikeChrome returns true if info looks like a handshake +// from a modern version of Chrome. +func (info rawHelloInfo) looksLikeChrome() bool { + // "We check for ciphers and extensions that Chrome is known + // to not support, but do not check for the inclusion of + // specific ciphers or extensions, nor do we validate their + // order. When appropriate, we check the presence and order + // of elliptic curves, compression methods, and EC point formats." + + // Not in Chrome 56, but present in Safari 10 (Feb. 2017): + // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 (0xc024) + // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 (0xc023) + // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (0xc00a) + // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (0xc009) + // TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 (0xc028) + // TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 (0xc027) + // TLS_RSA_WITH_AES_256_CBC_SHA256 (0x3d) + // TLS_RSA_WITH_AES_128_CBC_SHA256 (0x3c) + + // Not in Chrome 56, but present in Firefox 51 (Feb. 2017): + // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (0xc00a) + // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (0xc009) + // TLS_DHE_RSA_WITH_AES_128_CBC_SHA (0x33) + // TLS_DHE_RSA_WITH_AES_256_CBC_SHA (0x39) + + // Selected ciphers present in Chrome mobile (Feb. 2017): + // 0xc00a, 0xc014, 0xc009, 0x9c, 0x9d, 0x2f, 0x35, 0xa + + chromeCipherExclusions := map[uint16]struct{}{ + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384: {}, // 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: {}, // 0xc023 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384: {}, // 0xc028 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: {}, // 0xc027 + TLS_RSA_WITH_AES_256_CBC_SHA256: {}, // 0x3d + tls.TLS_RSA_WITH_AES_128_CBC_SHA256: {}, // 0x3c + TLS_DHE_RSA_WITH_AES_128_CBC_SHA: {}, // 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA: {}, // 0x39 + } + for _, ext := range info.cipherSuites { + if _, ok := chromeCipherExclusions[ext]; ok { + return false + } + } + + // Chrome does not include curve 25 (CurveP521) (as of Chrome 56, Feb. 2017). + for _, curve := range info.curves { + if curve == 25 { + return false + } + } + + return true +} + +// looksLikeEdge returns true if info looks like a handshake +// from a modern version of MS Edge. +func (info rawHelloInfo) looksLikeEdge() bool { + // "SChannel connections can by uniquely identified because SChannel + // is the only TLS library we tested that includes the OCSP status + // request extension before the supported groups and EC point formats + // extensions." + // + // More specifically, the OCSP status request extension appears + // *directly* before the other two extensions, which occur in that + // order. (I contacted the authors for clarification and verified it.) + for i, ext := range info.extensions { + if ext == extensionOCSPStatusRequest { + if len(info.extensions) <= i+2 { + return false + } + if info.extensions[i+1] != extensionSupportedCurves || + info.extensions[i+2] != extensionSupportedPoints { + return false + } + } + } + + for _, cs := range info.cipherSuites { + // As of Feb. 2017, Edge does not have 0xff, but Avast adds it + if cs == scsvRenegotiation { + return false + } + // Edge and modern IE do not have 0x4 or 0x5, but Blue Coat does + if cs == TLS_RSA_WITH_RC4_128_MD5 || cs == tls.TLS_RSA_WITH_RC4_128_SHA { + return false + } + } + + return true +} + +// looksLikeSafari returns true if info looks like a handshake +// from a modern version of MS Safari. +func (info rawHelloInfo) looksLikeSafari() bool { + // "One unique aspect of Secure Transport is that it includes + // the TLS_EMPTY_RENEGOTIATION_INFO_SCSV (0xff) cipher first, + // whereas the other libraries we investigated include the + // cipher last. Similar to Microsoft, Apple has changed + // TLS behavior in minor OS updates, which are not indicated + // in the HTTP User-Agent header. We allow for any of the + // updates when validating handshakes, and we check for the + // presence and ordering of ciphers, extensions, elliptic + // curves, and compression methods." + + // Note that any C lib (e.g. curl) compiled on macOS + // will probably use Secure Transport which will also + // share the TLS handshake characteristics of Safari. + + // Let's do the easy check first... should be sufficient in many cases. + if len(info.cipherSuites) < 1 { + return false + } + if info.cipherSuites[0] != scsvRenegotiation { + return false + } + + // We check for the presence and order of the extensions. + requiredExtensionsOrder := []uint16{10, 11, 13, 13172, 16, 5, 18, 23} + if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) { + return false + } + + // We check for order and presence of cipher suites + expectedCipherSuiteOrder := []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, // 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // 0xc023 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // 0xc00a + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, // 0xc028 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // 0xc027 + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // 0x9d + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // 0x9c + TLS_RSA_WITH_AES_256_CBC_SHA256, // 0x3d + TLS_RSA_WITH_AES_128_CBC_SHA256, // 0x3c + tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 + tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f + } + return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, true) +} + +// assertPresenceAndOrdering will return true if candidateList contains +// the items in requiredItems in the same order as requiredItems. +// +// If requiredIsSubset is true, then all items in requiredItems must be +// present in candidateList. If requiredIsSubset is false, then requiredItems +// may contain items that are not in candidateList. +// +// In all cases, the order of requiredItems is enforced. +func assertPresenceAndOrdering(requiredItems, candidateList []uint16, requiredIsSubset bool) bool { + superset := requiredItems + subset := candidateList + if requiredIsSubset { + superset = candidateList + subset = requiredItems + } + + var j int + for _, item := range subset { + var found bool + for j < len(superset) { + if superset[j] == item { + found = true + break + } + j++ + } + if j == len(superset) && !found { + return false + } + } + return true +} + +const ( + extensionOCSPStatusRequest = 5 + extensionSupportedCurves = 10 // also called "SupportedGroups" + extensionSupportedPoints = 11 + extensionHeartbeat = 15 + + scsvRenegotiation = 0xff + + // cipher suites missing from the crypto/tls package, + // in no particular order here + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xc023 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xc028 + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x3c + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x3d + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x39 + TLS_RSA_WITH_RC4_128_MD5 = 0x4 +) diff --git a/caddyhttp/httpserver/mitm_test.go b/caddyhttp/httpserver/mitm_test.go new file mode 100644 index 00000000..e5c75af8 --- /dev/null +++ b/caddyhttp/httpserver/mitm_test.go @@ -0,0 +1,206 @@ +package httpserver + +import ( + "crypto/tls" + "encoding/hex" + "reflect" + "testing" +) + +func TestParseClientHello(t *testing.T) { + for i, test := range []struct { + inputHex string + expected rawHelloInfo + }{ + { + // curl 7.51.0 (x86_64-apple-darwin16.0) libcurl/7.51.0 SecureTransport zlib/1.2.8 + inputHex: `010000a6030358a28c73a71bdfc1f09dee13fecdc58805dcce42ac44254df548f14645f7dc2c00004400ffc02cc02bc024c023c00ac009c008c030c02fc028c027c014c013c012009f009e006b0067003900330016009d009c003d003c0035002f000a00af00ae008d008c008b01000039000a00080006001700180019000b00020100000d00120010040102010501060104030203050306030005000501000000000012000000170000`, + expected: rawHelloInfo{ + cipherSuites: []uint16{255, 49196, 49195, 49188, 49187, 49162, 49161, 49160, 49200, 49199, 49192, 49191, 49172, 49171, 49170, 159, 158, 107, 103, 57, 51, 22, 157, 156, 61, 60, 53, 47, 10, 175, 174, 141, 140, 139}, + extensions: []uint16{10, 11, 13, 5, 18, 23}, + compressionMethods: []byte{0}, + curves: []tls.CurveID{23, 24, 25}, + points: []uint8{0}, + }, + }, + { + // Chrome 56 + inputHex: `010000c003031dae75222dae1433a5a283ddcde8ddabaefbf16d84f250eee6fdff48cdfff8a00000201a1ac02bc02fc02cc030cca9cca8cc14cc13c013c014009c009d002f0035000a010000777a7a0000ff010001000000000e000c0000096c6f63616c686f73740017000000230000000d00140012040308040401050308050501080606010201000500050100000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a000a0008aaaa001d001700182a2a000100`, + expected: rawHelloInfo{ + cipherSuites: []uint16{6682, 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49171, 49172, 156, 157, 47, 53, 10}, + extensions: []uint16{31354, 65281, 0, 23, 35, 13, 5, 18, 16, 30032, 11, 10, 10794}, + compressionMethods: []byte{0}, + curves: []tls.CurveID{43690, 29, 23, 24}, + points: []uint8{0}, + }, + }, + { + // Firefox 51 + inputHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`, + expected: rawHelloInfo{ + cipherSuites: []uint16{49195, 49199, 52393, 52392, 49196, 49200, 49162, 49161, 49171, 49172, 51, 57, 47, 53, 10}, + extensions: []uint16{0, 23, 65281, 10, 11, 35, 16, 5, 65283, 13}, + compressionMethods: []byte{0}, + curves: []tls.CurveID{29, 23, 24, 25}, + points: []uint8{0}, + }, + }, + { + // openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016) + inputHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`, + expected: rawHelloInfo{ + cipherSuites: []uint16{49200, 49196, 49192, 49188, 49172, 49162, 165, 163, 161, 159, 107, 106, 105, 104, 57, 56, 55, 54, 136, 135, 134, 133, 49202, 49198, 49194, 49190, 49167, 49157, 157, 61, 53, 132, 49199, 49195, 49191, 49187, 49171, 49161, 164, 162, 160, 158, 103, 64, 63, 62, 51, 50, 49, 48, 154, 153, 152, 151, 69, 68, 67, 66, 49201, 49197, 49193, 49189, 49166, 49156, 156, 60, 47, 150, 65, 7, 49169, 49159, 49164, 49154, 5, 4, 49170, 49160, 22, 19, 16, 13, 49165, 49155, 10, 255}, + extensions: []uint16{11, 10, 35, 13, 15}, + compressionMethods: []byte{1, 0}, + curves: []tls.CurveID{23, 25, 28, 27, 24, 26, 22, 14, 13, 11, 12, 9, 10}, + points: []uint8{0, 1, 2}, + }, + }, + } { + data, err := hex.DecodeString(test.inputHex) + if err != nil { + t.Fatalf("Test %d: Could not decode hex data: %v", i, err) + } + actual := parseRawClientHello(data) + if !reflect.DeepEqual(test.expected, actual) { + t.Errorf("Test %d: Expected %+v; got %+v", i, test.expected, actual) + } + } +} + +func TestHeuristicFunctions(t *testing.T) { + // To test the heuristics, we assemble a collection of real + // ClientHello messages from various TLS clients. Please be + // sure to hex-encode them and document the User-Agent + // associated with the connection. + // + // If the TLS client used is not an HTTP client (e.g. s_client), + // you can leave the userAgent blank, but please use a comment + // to document crucial missing information such as client name, + // version, and platform, maybe even the date you collected + // the sample! Please group similar clients together, ordered + // by version for convenience. + + // clientHello pairs a User-Agent string to its ClientHello message. + type clientHello struct { + userAgent string + helloHex string + } + + // clientHellos groups samples of true (real) ClientHellos by the + // name of the browser that produced them. We limit the set of + // browsers to those we are programmed to protect, as well as a + // category for "Other" which contains real ClientHello messages + // from clients that we do not recognize, which may be used to + // test or imitate interception scenarios. + // + // Please group similar clients and order by version for convenience + // when adding to the test cases. + clientHellos := map[string][]clientHello{ + "Chrome": { + { + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + helloHex: `010000c003031dae75222dae1433a5a283ddcde8ddabaefbf16d84f250eee6fdff48cdfff8a00000201a1ac02bc02fc02cc030cca9cca8cc14cc13c013c014009c009d002f0035000a010000777a7a0000ff010001000000000e000c0000096c6f63616c686f73740017000000230000000d00140012040308040401050308050501080606010201000500050100000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a000a0008aaaa001d001700182a2a000100`, + }, + }, + "Firefox": { + { + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:51.0) Gecko/20100101 Firefox/51.0", + helloHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`, + }, + }, + "Edge": { + { + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000bd030358a3c9bf05f734842e189fb6ce653b67b846e990bc1fc5fb8c397874d06020f1000038c02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a00400038003200130100005c000500050100000000000a00080006001d00170018000b00020100000d00140012040105010201040305030203020206010603002300000010000e000c02683208687474702f312e310017000055000006000100020002ff01000100`, + }, + }, + "Safari": { + { + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8", + helloHex: `010000d2030358a295b513c8140c6ff880f4a8a73cc830ed2dab2c4f2068eb365228d828732e00002600ffc02cc02bc024c023c00ac009c030c02fc028c027c014c013009d009c003d003c0035002f010000830000000e000c0000096c6f63616c686f7374000a00080006001700180019000b00020100000d00120010040102010501060104030203050306033374000000100030002e0268320568322d31360568322d31350568322d313408737064792f332e3106737064792f3308687474702f312e310005000501000000000012000000170000`, + }, + }, + "Other": { // these are either non-browser clients or intercepted client hellos + { + // openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016) + helloHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`, + }, + { + // curl 7.51.0 (x86_64-apple-darwin16.0) libcurl/7.51.0 SecureTransport zlib/1.2.8 + userAgent: "curl/7.51.0", + helloHex: `010000a6030358a28c73a71bdfc1f09dee13fecdc58805dcce42ac44254df548f14645f7dc2c00004400ffc02cc02bc024c023c00ac009c008c030c02fc028c027c014c013c012009f009e006b0067003900330016009d009c003d003c0035002f000a00af00ae008d008c008b01000039000a00080006001700180019000b00020100000d00120010040102010501060104030203050306030005000501000000000012000000170000`, + }, + { + // Avast 17.1.2286 (Feb. 2017) on Windows 10 x64 build 14393, intercepting Edge + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000ce0303b418fdc4b6cf6436a5e2bfb06b96ed5faa7285c20c7b49341a78be962a9dc40000003ac02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a004000380032001300ff0100006b00000014001200000f66696e6572706978656c732e636f6d000b000403000102000a00080006001d0017001800230000000d001400120401050102010403050302030202060106030005000501000000000010000e000c02683208687474702f312e310016000000170000`, + }, + { + // Kaspersky Internet Security 17.0.0.611 on Windows 10 x64 build 14393, intercepting Edge + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000eb030361ce302bf4b0d5adf1ff30b2cf433c4a4b68f33e07b2651695e7ae6ec3cf126400003ac02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a004000380032001300ff0100008800000014001200000f66696e6572706978656c732e636f6d000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000500050100000000000f0001010010000e000c02683208687474702f312e31`, + }, + { + // Kaspersky Internet Security 17.0.0.611 on Windows 10 x64 build 14393, intercepting Firefox 51 + userAgent: "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0", + helloHex: `010001fc0303768e3f9ea75194c7cb03d23e8e6371b95fb696d339b797be57a634309ec98a42200f2a7554098364b7f05d21a8c7f43f31a893a4fc5670051020408c8e4dc234dd001cc02bc02fc02cc030c00ac009c013c01400330039002f0035000a00ff0100019700000014001200000f66696e6572706978656c732e636f6d000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230078bf4e244d4de3d53c6331edda9672dfc4a17aae92b671e86da1368b1b5ae5324372817d8f3b7ffe1a7a1537a5049b86cd7c44863978c1e615b005942755da20fc3a4e34a16f78034aa3b1cffcef95f81a0995c522a53b0e95a4f98db84c43359d93d8647b2de2a69f3ebdcfc6bca452730cbd00179226dedf000d0020001e060106020603050105020503040104020403030103020303020102020203000500050100000000000f0001010010000e000c02683208687474702f312e3100150093000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000`, + }, + { + // Kaspersky Internet Security 17.0.0.611 on Windows 10 x64 build 14393, intercepting Chrome 56 + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + helloHex: `010000c903033481e7af24e647ba5a79ec97e9264c1a1f990cf842f50effe22be52130d5af82000018c02bc02fc02cc030c013c014009c009d002f0035000a00ff0100008800000014001200000f66696e6572706978656c732e636f6d000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000500050100000000000f0001010010000e000c02683208687474702f312e31`, + }, + { + // AVG 17.1.3006 (build 17.1.3354.20) on Windows 10 x64 build 14393, intercepting Edge + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000ca0303fd83091207161eca6b4887db50587109c50e463beb190362736b1fcf9e05f807000036c02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f006a00400038003200ff0100006b00000014001200000f66696e6572706978656c732e636f6d000b000403000102000a00080006001d0017001800230000000d001400120401050102010403050302030202060106030005000501000000000010000e000c02683208687474702f312e310016000000170000`, + }, + { + // IE 11 on Windows 7, this connection was intercepted by Blue Coat + helloHex: "010000b1030358a3f3bae627f464da8cb35976b88e9119640032d41e62a107d608ed8d3e62b9000034c028c027c014c013009f009e009d009cc02cc02bc024c023c00ac009003d003c0035002f006a004000380032000a0013000500040100005400000014001200000f66696e6572706978656c732e636f6d000500050100000000000a00080006001700180019000b00020100000d0014001206010603040105010201040305030203020200170000ff01000100", + }, + }, + } + + for client, chs := range clientHellos { + for i, ch := range chs { + hello, err := hex.DecodeString(ch.helloHex) + if err != nil { + t.Errorf("[%s] Test %d: Error decoding ClientHello: %v", client, i, err) + continue + } + parsed := parseRawClientHello(hello) + + isChrome := parsed.looksLikeChrome() + isFirefox := parsed.looksLikeFirefox() + isSafari := parsed.looksLikeSafari() + isEdge := parsed.looksLikeEdge() + + // we want each of the heuristic functions to be as + // exclusive but as low-maintenance as possible; + // in other words, if one returns true, the others + // should return false, with as little logic as possible, + // but with enough logic to force TLS proxies to do a + // good job preserving characterstics of the handshake. + var correct bool + switch client { + case "Chrome": + correct = isChrome && !isFirefox && !isSafari && !isEdge + case "Firefox": + correct = !isChrome && isFirefox && !isSafari && !isEdge + case "Safari": + correct = !isChrome && !isFirefox && isSafari && !isEdge + case "Edge": + correct = !isChrome && !isFirefox && !isSafari && isEdge + case "Other": + correct = !isChrome && !isFirefox && !isSafari && !isEdge + } + + if !correct { + t.Errorf("[%s] Test %d: Chrome=%v, Firefox=%v, Safari=%v, Edge=%v; parsed hello: %+v", + client, i, isChrome, isFirefox, isSafari, isEdge, parsed) + } + } + } +} diff --git a/caddyhttp/httpserver/replacer.go b/caddyhttp/httpserver/replacer.go index 22e8aa8f..395aaacc 100644 --- a/caddyhttp/httpserver/replacer.go +++ b/caddyhttp/httpserver/replacer.go @@ -298,6 +298,15 @@ func (r *replacer) getSubstitution(key string) string { } } return requestReplacer.Replace(r.requestBody.String()) + case "{mitm}": + if val, ok := r.request.Context().Value(CtxKey("mitm")).(bool); ok { + if val { + return "likely" + } else { + return "unlikely" + } + } + return "unknown" case "{status}": if r.responseRecorder == nil { return r.emptyValue diff --git a/caddyhttp/httpserver/server.go b/caddyhttp/httpserver/server.go index b686b5d7..7345a0d7 100644 --- a/caddyhttp/httpserver/server.go +++ b/caddyhttp/httpserver/server.go @@ -47,6 +47,18 @@ func NewServer(addr string, group []*SiteConfig) (*Server, error) { } s.Server.Handler = s // this is weird, but whatever + tlsh := &tlsHandler{next: s.Server.Handler} + s.Server.ConnState = func(c net.Conn, cs http.ConnState) { + // when a connection closes or is hijacked, delete its entry + // in the map, because we are done with it. + if tlsh.listener != nil { + if cs == http.StateHijacked || cs == http.StateClosed { + tlsh.listener.helloInfosMu.Lock() + delete(tlsh.listener.helloInfos, c.RemoteAddr().String()) + tlsh.listener.helloInfosMu.Unlock() + } + } + } // Disable HTTP/2 if desired if !HTTP2 { @@ -75,6 +87,10 @@ func NewServer(addr string, group []*SiteConfig) (*Server, error) { s.Server.TLSConfig.NextProtos = []string{"h2"} } + if s.Server.TLSConfig != nil { + s.Server.Handler = tlsh + } + // Compile custom middleware for every site (enables virtual hosting) for _, site := range group { stack := Handler(staticfiles.FileServer{Root: http.Dir(site.Root), Hide: site.HiddenFiles}) @@ -156,7 +172,10 @@ func (s *Server) Serve(ln net.Listener) error { // not implement the File() method we need for graceful restarts // on POSIX systems. // TODO: Is this ^ still relevant anymore? Maybe we can now that it's a net.Listener... - ln = tls.NewListener(ln, s.Server.TLSConfig) + ln = newTLSListener(ln, s.Server.TLSConfig, s.Server.ReadTimeout) + if handler, ok := s.Server.Handler.(*tlsHandler); ok { + handler.listener = ln.(*tlsHelloListener) + } // Rotate TLS session ticket keys s.tlsGovChan = caddytls.RotateSessionTicketKeys(s.Server.TLSConfig)