Enable TLS based communication with Zookeeper Backend (#4856)

* The added method customTLSDial() creates a tls connection to the zookeeper backend when 'tls_enabled' is set to true in config

* Update to the document for TLS configuration that is  required to enable TLS connection to Zookeeper backend

* Minor formatting update

* Minor update to the description for example config

* As per review comments from @kenbreeman, additional property description indicating support for multiple Root CAs in a single file has been added

* minor formatting

physical/zookeeper/zookeeper.go

@@ -2,7 +2,11 @@ package zookeeper
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
+	"net"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -11,9 +15,11 @@ import (
 
 	"github.com/hashicorp/errwrap"
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/parseutil"
 	"github.com/hashicorp/vault/physical"
 
 	metrics "github.com/armon/go-metrics"
+	"github.com/hashicorp/vault/helper/tlsutil"
 	"github.com/samuel/go-zookeeper/zk"
 )
 
@@ -119,7 +125,7 @@ func NewZooKeeperBackend(conf map[string]string, logger log.Logger) (physical.Ba
 	}
 
 	// We have all of the configuration in hand - let's try and connect to ZK
-	client, _, err := zk.Connect(strings.Split(machines, ","), time.Second)
+	client, _, err := createClient(conf, machines, time.Second)
 	if err != nil {
 		return nil, errwrap.Wrapf("client setup failed: {{err}}", err)
 	}
@@ -142,6 +148,162 @@ func NewZooKeeperBackend(conf map[string]string, logger log.Logger) (physical.Ba
 	return c, nil
 }
 
+func caseInsenstiveContains(superset, val string) bool {
+	return strings.Contains(strings.ToUpper(superset), strings.ToUpper(val))
+}
+
+// Returns a client for ZK connection. Config value 'tls_enabled' determines if TLS is enabled or not.
+func createClient(conf map[string]string, machines string, timeout time.Duration) (*zk.Conn, <-chan zk.Event, error) {
+	// 'tls_enabled' defaults to false
+	isTlsEnabled := false
+	isTlsEnabledStr, ok := conf["tls_enabled"]
+
+	if ok && isTlsEnabledStr != "" {
+		parsedBoolval, err := parseutil.ParseBool(isTlsEnabledStr)
+		if err != nil {
+			return nil, nil, errwrap.Wrapf("failed parsing tls_enabled parameter: {{err}}", err)
+		}
+		isTlsEnabled = parsedBoolval
+	}
+
+	if isTlsEnabled {
+		// Create a custom Dialer with cert configuration for TLS handshake.
+		tlsDialer := customTLSDial(conf, machines)
+		options := zk.WithDialer(tlsDialer)
+		return zk.Connect(strings.Split(machines, ","), timeout, options)
+	} else {
+		return zk.Connect(strings.Split(machines, ","), timeout)
+	}
+}
+
+// Vault config file properties:
+// 1. tls_skip_verify: skip host name verification.
+// 2. tls_min_version: minimum supported/acceptable tls version
+// 3. tls_cert_file: Cert file Absolute path
+// 4. tls_key_file: Key file Absolute path
+// 5. tls_ca_file: ca file absolute path
+// 6. tls_verify_ip: If set to true, server's IP is verified in certificate if tls_skip_verify is false.
+func customTLSDial(conf map[string]string, machines string) zk.Dialer {
+	return func(network, addr string, timeout time.Duration) (net.Conn, error) {
+		// Sets the serverName. *Note* the addr field comes in as an IP address
+		serverName, _, sParseErr := net.SplitHostPort(addr)
+		if sParseErr != nil {
+			// If the address is only missing port, assign the full address anyway
+			if strings.Contains(sParseErr.Error(), "missing port") {
+				serverName = addr
+			} else {
+				return nil, errwrap.Wrapf("failed parsing the server address for 'serverName' setting {{err}}", sParseErr)
+			}
+		}
+
+		insecureSkipVerify := false
+		tlsSkipVerify, ok := conf["tls_skip_verify"]
+
+		if ok && tlsSkipVerify != "" {
+			b, err := parseutil.ParseBool(tlsSkipVerify)
+			if err != nil {
+				return nil, errwrap.Wrapf("failed parsing tls_skip_verify parameter: {{err}}", err)
+			}
+			insecureSkipVerify = b
+		}
+
+		if !insecureSkipVerify {
+			// If tls_verify_ip is set to false, Server's DNS name is verified in the CN/SAN of the certificate.
+			// if tls_verify_ip is true, Server's IP is verified in the CN/SAN of the certificate.
+			// These checks happen only when tls_skip_verify is set to false.
+			// This value defaults to false
+			ipSanCheck := false
+			configVal, lookupOk := conf["tls_verify_ip"]
+
+			if lookupOk && configVal != "" {
+				parsedIpSanCheck, ipSanErr := parseutil.ParseBool(configVal)
+				if ipSanErr != nil {
+					return nil, errwrap.Wrapf("failed parsing tls_verify_ip parameter: {{err}}", ipSanErr)
+				}
+				ipSanCheck = parsedIpSanCheck
+			}
+			// The addr/serverName parameter to this method comes in as an IP address.
+			// Here we lookup the DNS name and assign it to serverName if ipSanCheck is set to false
+			if !ipSanCheck {
+				lookupAddressMany, lookupErr := net.LookupAddr(serverName)
+				if lookupErr == nil {
+					for _, lookupAddress := range lookupAddressMany {
+						// strip the trailing '.' from lookupAddr
+						if lookupAddress[len(lookupAddress)-1] == '.' {
+							lookupAddress = lookupAddress[:len(lookupAddress)-1]
+						}
+						// Allow serverName to be replaced only if the lookupname is part of the
+						// supplied machine names
+						// If there is no match, the serverName will continue to be an IP value.
+						if caseInsenstiveContains(machines, lookupAddress) {
+							serverName = lookupAddress
+							break
+						}
+					}
+				}
+			}
+
+		}
+
+		tlsMinVersionStr, ok := conf["tls_min_version"]
+		if !ok {
+			// Set the default value
+			tlsMinVersionStr = "tls12"
+		}
+
+		tlsMinVersion, ok := tlsutil.TLSLookup[tlsMinVersionStr]
+		if !ok {
+			return nil, fmt.Errorf("invalid 'tls_min_version'")
+		}
+
+		tlsClientConfig := &tls.Config{
+			MinVersion:         tlsMinVersion,
+			InsecureSkipVerify: insecureSkipVerify,
+			ServerName:         serverName,
+		}
+
+		_, okCert := conf["tls_cert_file"]
+		_, okKey := conf["tls_key_file"]
+
+		if okCert && okKey {
+			tlsCert, err := tls.LoadX509KeyPair(conf["tls_cert_file"], conf["tls_key_file"])
+			if err != nil {
+				return nil, errwrap.Wrapf("client tls setup failed for ZK: {{err}}", err)
+			}
+
+			tlsClientConfig.Certificates = []tls.Certificate{tlsCert}
+		}
+
+		if tlsCaFile, ok := conf["tls_ca_file"]; ok {
+			caPool := x509.NewCertPool()
+
+			data, err := ioutil.ReadFile(tlsCaFile)
+			if err != nil {
+				return nil, errwrap.Wrapf("failed to read ZK CA file: {{err}}", err)
+			}
+
+			if !caPool.AppendCertsFromPEM(data) {
+				return nil, fmt.Errorf("failed to parse ZK CA certificate")
+			}
+			tlsClientConfig.RootCAs = caPool
+		}
+
+		if network != "tcp" {
+			return nil, fmt.Errorf("unsupported network %q", network)
+		}
+
+		tcpConn, err := net.DialTimeout("tcp", addr, timeout)
+		if err != nil {
+			return nil, err
+		}
+		conn := tls.Client(tcpConn, tlsClientConfig)
+		if err := conn.Handshake(); err != nil {
+			return nil, fmt.Errorf("Handshake failed with Zookeeper : %v", err)
+		}
+		return conn, nil
+	}
+}
+
 // ensurePath is used to create each node in the path hierarchy.
 // We avoid calling this optimistically, and invoke it when we get
 // an error during an operation

website/source/docs/configuration/storage/zookeeper.html.md

@@ -66,6 +66,30 @@ znodes and, potentially, take Vault out of service.
     ip:70.95.0.0/16
     ```
 
+- `tls_enabled` `(bool: false)` – Specifies if TLS communication with the Zookeeper
+   backend has to be enabled.
+
+- `tls_ca_file` `(string: "")` – Specifies the path to the CA certificate file used
+  for Zookeeper communication. Multiple CA certificates can be provided in the same file.
+
+- `tls_cert_file` `(string: "")` (optional) – Specifies the path to the
+  client certificate for Zookeeper communication.
+
+- `tls_key_file` `(string: "")` – Specifies the path to the private key for
+  Zookeeper communication.
+
+- `tls_min_version` `(string: "tls12")` – Specifies the minimum TLS version to
+  use. Accepted values are `"tls10"`, `"tls11"` or `"tls12"`.
+
+- `tls_skip_verify` `(bool: false)` – Specifies if the TLS host verification
+  should be disabled. It is highly discouraged that you disable this option.
+
+- `tls_verify_ip` `(bool: false)` - This property comes into play only when
+  'tls_skip_verify' is set to false. When 'tls_verify_ip' is set to 'true', the
+  zookeeper server's IP is verified in the presented certificates CN/SAN entry. 
+  When set to 'false' the server's DNS name is verified in the certificates CN/SAN entry.
+
+
 ## `zookeeper` Examples
 
 ### Custom Address and Path
@@ -105,4 +129,25 @@ storage "zookeeper" {
 }
 ```
 
+### zNode connection over TLS.
+
+This example instructs Vault to connect to Zookeeper using the provided TLS configuration. The host verification will happen with the presented certificate using the servers IP because 'tls_verify_ip' is set to true.
+
+```hcl
+storage "zookeeper" {
+  address = "host1.com:5200,host2.com:5200,host3.com:5200"
+  path = "vault_path_on_zk/"
+  znode_owner = "digest:vault_user:digestvalueforpassword="
+  auth_info = "digest:vault_user:thisisthepassword"
+  redirect_addr = "http://localhost:8200"
+  tls_verify_ip = "true"
+  tls_enabled= "true"
+  tls_min_version= "tls12"
+  tls_cert_file = "/path/to/the/cert/file/zkcert.pem"
+  tls_key_file = "/path/to/the/key/file/zkkey.pem"
+  tls_skip_verify= "false"
+  tls_ca_file= "/path/to/the/ca/file/ca.pem"
+}
+```
+
 [zk]: https://zookeeper.apache.org/