RPC encryption

client/client.go

@@ -16,6 +16,7 @@ import (
 	"github.com/armon/go-metrics"
 	consulapi "github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/lib"
+	"github.com/hashicorp/consul/tlsutil"
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
@@ -155,13 +156,14 @@ type Client struct {
 }
 
 // NewClient is used to create a new client from the given configuration
-func NewClient(cfg *config.Config, consulSyncer *consul.Syncer, logger *log.Logger) (*Client, error) {
+func NewClient(cfg *config.Config, consulSyncer *consul.Syncer, logger *log.Logger,
+	tlsWrap tlsutil.DCWrapper) (*Client, error) {
 	// Create the client
 	c := &Client{
 		config:             cfg,
 		consulSyncer:       consulSyncer,
 		start:              time.Now(),
-		connPool:           nomad.NewPool(cfg.LogOutput, clientRPCCache, clientMaxStreams, nil),
+		connPool:           nomad.NewPool(cfg.LogOutput, clientRPCCache, clientMaxStreams, tlsWrap),
 		logger:             logger,
 		hostStatsCollector: stats.NewHostStatsCollector(),
 		allocs:             make(map[string]*AllocRunner),

client/client_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/tlsutil"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/command/agent/consul"
 	"github.com/hashicorp/nomad/nomad"
@@ -83,7 +84,7 @@ func testServer(t *testing.T, cb func(*nomad.Config)) (*nomad.Server, string) {
 	return server, config.RPCAddr.String()
 }
 
-func testClient(t *testing.T, cb func(c *config.Config)) *Client {
+func testClient(t *testing.T, cb func(c *config.Config), tlsWrap tlsutil.DCWrapper) *Client {
 	conf := config.DefaultConfig()
 	conf.VaultConfig.Enabled = false
 	conf.DevMode = true
@@ -98,15 +99,25 @@ func testClient(t *testing.T, cb func(c *config.Config)) *Client {
 	}
 
 	logger := log.New(conf.LogOutput, "", log.LstdFlags)
-	client, err := NewClient(conf, consulSyncer, logger)
+	client, err := NewClient(conf, consulSyncer, logger, tlsWrap)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
 	return client
 }
 
+func configureTLS(config *nomad.Config, mode string) {
+	config.VerifyIncoming = true
+	config.VerifyOutgoing = true
+	config.CAFile = "../test/ca/root.cer"
+	config.CertFile = fmt.Sprintf("../test/key/%s.cer", mode)
+	config.KeyFile = fmt.Sprintf("../test/key/%s.key", mode)
+	config.Domain = "internal"
+	config.Region = "test"
+}
+
 func TestClient_StartStop(t *testing.T) {
-	client := testClient(t, nil)
+	client := testClient(t, nil, nil)
 	if err := client.Shutdown(); err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -118,7 +129,36 @@ func TestClient_RPC(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.Servers = []string{addr}
+	}, nil)
+	defer c1.Shutdown()
+
+	// RPC should succeed
+	testutil.WaitForResult(func() (bool, error) {
+		var out struct{}
+		err := c1.RPC("Status.Ping", struct{}{}, &out)
+		return err == nil, err
+	}, func(err error) {
+		t.Fatalf("err: %v", err)
+	})
+}
+
+func TestClient_RPC_TLS(t *testing.T) {
+	s1, addr := testServer(t, func(c *nomad.Config) {
+		configureTLS(c, "server")
 	})
+	defer s1.Shutdown()
+
+	conf := nomad.DefaultConfig()
+	configureTLS(conf, "client")
+	tlsWrap, err := conf.TlsConfig().OutgoingTLSWrapper()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	c1 := testClient(t, func(c *config.Config) {
+		c.Servers = []string{addr}
+		c.Region = "test"
+	}, tlsWrap)
 	defer c1.Shutdown()
 
 	// RPC should succeed
@@ -137,7 +177,7 @@ func TestClient_RPC_Passthrough(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	// RPC should succeed
@@ -151,7 +191,7 @@ func TestClient_RPC_Passthrough(t *testing.T) {
 }
 
 func TestClient_Fingerprint(t *testing.T) {
-	c := testClient(t, nil)
+	c := testClient(t, nil, nil)
 	defer c.Shutdown()
 
 	// Ensure kernel and arch are always present
@@ -165,7 +205,7 @@ func TestClient_Fingerprint(t *testing.T) {
 }
 
 func TestClient_HasNodeChanged(t *testing.T) {
-	c := testClient(t, nil)
+	c := testClient(t, nil, nil)
 	defer c.Shutdown()
 
 	node := c.Node()
@@ -203,7 +243,7 @@ func TestClient_Fingerprint_InWhitelist(t *testing.T) {
 
 		// Weird spacing to test trimming. Whitelist all modules expect cpu.
 		c.Options["fingerprint.whitelist"] = "  arch, consul,cpu,env_aws,env_gce,host,memory,network,storage,foo,bar	"
-	})
+	}, nil)
 	defer c.Shutdown()
 
 	node := c.Node()
@@ -219,7 +259,7 @@ func TestClient_Fingerprint_OutOfWhitelist(t *testing.T) {
 		}
 
 		c.Options["fingerprint.whitelist"] = "arch,consul,env_aws,env_gce,host,memory,network,storage,foo,bar"
-	})
+	}, nil)
 	defer c.Shutdown()
 
 	node := c.Node()
@@ -229,7 +269,7 @@ func TestClient_Fingerprint_OutOfWhitelist(t *testing.T) {
 }
 
 func TestClient_Drivers(t *testing.T) {
-	c := testClient(t, nil)
+	c := testClient(t, nil, nil)
 	defer c.Shutdown()
 
 	node := c.Node()
@@ -250,7 +290,7 @@ func TestClient_Drivers_InWhitelist(t *testing.T) {
 
 		// Weird spacing to test trimming
 		c.Options["driver.whitelist"] = "   exec ,  foo	"
-	})
+	}, nil)
 	defer c.Shutdown()
 
 	node := c.Node()
@@ -270,7 +310,7 @@ func TestClient_Drivers_OutOfWhitelist(t *testing.T) {
 		}
 
 		c.Options["driver.whitelist"] = "foo,bar,baz"
-	})
+	}, nil)
 	defer c.Shutdown()
 
 	node := c.Node()
@@ -286,7 +326,7 @@ func TestClient_Register(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	req := structs.NodeSpecificRequest{
@@ -319,7 +359,7 @@ func TestClient_Heartbeat(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	req := structs.NodeSpecificRequest{
@@ -350,7 +390,7 @@ func TestClient_UpdateAllocStatus(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	// Wait til the node is ready
@@ -399,7 +439,7 @@ func TestClient_WatchAllocs(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	// Wait til the node is ready
@@ -498,7 +538,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
 	c1 := testClient(t, func(c *config.Config) {
 		c.DevMode = false
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	// Wait til the node is ready
@@ -554,7 +594,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	c2, err := NewClient(c1.config, consulSyncer, logger)
+	c2, err := NewClient(c1.config, consulSyncer, logger, nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -615,7 +655,7 @@ func TestClient_BlockedAllocations(t *testing.T) {
 
 	c1 := testClient(t, func(c *config.Config) {
 		c.RPCHandler = s1
-	})
+	}, nil)
 	defer c1.Shutdown()
 
 	// Wait for the node to be ready

command/agent/agent.go

@@ -133,6 +133,14 @@ func (a *Agent) serverConfig() (*nomad.Config, error) {
 		conf.EnabledSchedulers = a.config.Server.EnabledSchedulers
 	}
 
+	// Copy the TLS configuration
+	conf.VerifyIncoming = a.config.VerifyIncoming
+	conf.VerifyOutgoing = a.config.VerifyOutgoing
+	conf.CAFile = a.config.CAFile
+	conf.CertFile = a.config.CertFile
+	conf.KeyFile = a.config.KeyFile
+	conf.Domain = a.config.Domain
+
 	// Set up the advertise addrs
 	if addr := a.config.AdvertiseAddrs.Serf; addr != "" {
 		serfAddr, err := net.ResolveTCPAddr("tcp", addr)
@@ -288,8 +296,8 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 	}
 	if len(invalidConsulKeys) > 0 {
 		a.logger.Printf("[WARN] agent: Invalid keys: %v", strings.Join(invalidConsulKeys, ","))
-		a.logger.Printf(`Nomad client ignores consul related configuration in client options. 
-		Please refer to the guide https://www.nomadproject.io/docs/agent/config.html#consul_options 
+		a.logger.Printf(`Nomad client ignores consul related configuration in client options.
+		Please refer to the guide https://www.nomadproject.io/docs/agent/config.html#consul_options
 		to configure Nomad to work with Consul.`)
 	}
 
@@ -450,8 +458,28 @@ func (a *Agent) setupClient() error {
 		}
 	}
 
+	// Create the tls wrapper for outgoing connections
+	nomadConf := a.config.NomadConfig
+	if nomadConf == nil {
+		nomadConf = nomad.DefaultConfig()
+	}
+	nomadConf.VerifyIncoming = a.config.VerifyIncoming
+	nomadConf.VerifyOutgoing = a.config.VerifyOutgoing
+	nomadConf.CAFile = a.config.CAFile
+	nomadConf.CertFile = a.config.CertFile
+	nomadConf.KeyFile = a.config.KeyFile
+	nomadConf.Domain = a.config.Domain
+
+	tlsConf := nomadConf.TlsConfig()
+
+	// Create the tls Wrapper
+	tlsWrap, err := tlsConf.OutgoingTLSWrapper()
+	if err != nil {
+		return err
+	}
+
 	// Create the client
-	client, err := client.NewClient(conf, a.consulSyncer, a.logger)
+	client, err := client.NewClient(conf, a.consulSyncer, a.logger, tlsWrap)
 	if err != nil {
 		return fmt.Errorf("client setup failed: %v", err)
 	}

command/agent/config.go

@@ -41,6 +41,34 @@ type Config struct {
 	// EnableDebug is used to enable debugging HTTP endpoints
 	EnableDebug bool `mapstructure:"enable_debug"`
 
+	// VerifyIncoming is used to verify the authenticity of incoming connections.
+	// This means that TCP requests are forbidden, only allowing for TLS. TLS connections
+	// must match a provided certificate authority. This can be used to force client auth.
+	VerifyIncoming bool `mapstructure:"verify_incoming"`
+
+	// VerifyOutgoing is used to verify the authenticity of outgoing connections.
+	// This means that TLS requests are used. TLS connections must match a provided
+	// certificate authority. This is used to verify authenticity of server nodes.
+	// This also ensures that the certificate presented is valid for server.<region>.<domain>
+	// in order to prevent a compromised client from being restarted as a server, and then
+	// intercepting request traffic as well as being added as a raft peer.
+	VerifyOutgoing bool `mapstructure:"verify_outgoing"`
+
+	// CAFile is a path to a certificate authority file. This is used with VerifyIncoming
+	// or VerifyOutgoing to verify the TLS connection.
+	CAFile string `mapstructure:"ca_file"`
+
+	// CertFile is used to provide a TLS certificate that is used for serving TLS connections.
+	// Must be provided to serve TLS connections.
+	CertFile string `mapstructure:"cert_file"`
+
+	// KeyFile is used to provide a TLS key that is used for serving TLS connections.
+	// Must be provided to serve TLS connections.
+	KeyFile string `mapstructure:"key_file"`
+
+	// This is used during CAFile verification if VerifyOutgoing is set to true
+	Domain string `mapstructure:"domain"`
+
 	// Ports is used to control the network ports we bind to.
 	Ports *Ports `mapstructure:"ports"`
 
@@ -442,6 +470,7 @@ func DefaultConfig() *Config {
 		LogLevel:   "INFO",
 		Region:     "global",
 		Datacenter: "dc1",
+		Domain:     "nomad",
 		BindAddr:   "127.0.0.1",
 		Ports: &Ports{
 			HTTP: 4646,
@@ -545,6 +574,26 @@ func (c *Config) Merge(b *Config) *Config {
 		result.DisableAnonymousSignature = true
 	}
 
+	// Apply TLS config
+	if b.VerifyIncoming {
+		result.VerifyIncoming = true
+	}
+	if b.VerifyOutgoing {
+		result.VerifyOutgoing = true
+	}
+	if b.CAFile != "" {
+		result.CAFile = b.CAFile
+	}
+	if b.CertFile != "" {
+		result.CertFile = b.CertFile
+	}
+	if b.KeyFile != "" {
+		result.KeyFile = b.KeyFile
+	}
+	if b.Domain != "" {
+		result.Domain = b.Domain
+	}
+
 	// Apply the telemetry config
 	if result.Telemetry == nil && b.Telemetry != nil {
 		telemetry := *b.Telemetry

command/agent/config_parse.go

@@ -95,6 +95,12 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 		"consul",
 		"vault",
 		"http_api_response_headers",
+		"verify_incoming",
+		"verify_outgoing",
+		"ca_file",
+		"cert_file",
+		"key_file",
+		"domain",
 	}
 	if err := checkHCLKeys(list, valid); err != nil {
 		return multierror.Prefix(err, "config:")