Enable serf encryption (#1791)

* Added the keygen command

* Added support for gossip encryption

* Changed the URL for keyring management

* Fixed the cli

* Added some tests

* Added tests for keyring operations

* Added a test for removal of keys

* Added some docs

* Fixed some docs

* Added general options

api/agent.go

@@ -16,6 +16,19 @@ type Agent struct {
 	region     string
 }
 
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	Messages map[string]string
+	Keys     map[string]int
+	NumNodes int
+}
+
+// KeyringRequest is request objects for serf key operations.
+type KeyringRequest struct {
+	Key string
+}
+
 // Agent returns a new agent which can be used to query
 // the agent-specific endpoints.
 func (c *Client) Agent() *Agent {
@@ -157,6 +170,46 @@ func (a *Agent) SetServers(addrs []string) error {
 	return err
 }
 
+// ListKeys returns the list of installed keys
+func (a *Agent) ListKeys() (*KeyringResponse, error) {
+	var resp KeyringResponse
+	_, err := a.client.query("/v1/agent/keyring/list", &resp, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// InstallKey installs a key in the keyrings of all the serf members
+func (a *Agent) InstallKey(key string) (*KeyringResponse, error) {
+	args := KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keyring/install", &args, &resp, nil)
+	return &resp, err
+}
+
+// UseKey uses a key from the keyring of serf members
+func (a *Agent) UseKey(key string) (*KeyringResponse, error) {
+	args := KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keyring/use", &args, &resp, nil)
+	return &resp, err
+}
+
+// RemoveKey removes a particular key from keyrings of serf members
+func (a *Agent) RemoveKey(key string) (*KeyringResponse, error) {
+	args := KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keyring/remove", &args, &resp, nil)
+	return &resp, err
+}
+
 // joinResponse is used to decode the response we get while
 // sending a member join request.
 type joinResponse struct {

command/agent/agent.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"log"
 	"net"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -371,6 +372,11 @@ func (a *Agent) setupServer() error {
 		return fmt.Errorf("server config setup failed: %s", err)
 	}
 
+	// Sets up the keyring for gossip encryption
+	if err := a.setupKeyrings(conf); err != nil {
+		return fmt.Errorf("failed to configure keyring: %v", err)
+	}
+
 	// Create the server
 	server, err := nomad.NewServer(conf, a.consulSyncer, a.logger)
 	if err != nil {
@@ -431,6 +437,30 @@ func (a *Agent) setupServer() error {
 	return nil
 }
 
+// setupKeyrings is used to initialize and load keyrings during agent startup
+func (a *Agent) setupKeyrings(config *nomad.Config) error {
+	file := filepath.Join(a.config.DataDir, serfKeyring)
+
+	if a.config.Server.EncryptKey == "" {
+		goto LOAD
+	}
+	if _, err := os.Stat(file); err != nil {
+		if err := initKeyring(file, a.config.Server.EncryptKey); err != nil {
+			return err
+		}
+	}
+
+LOAD:
+	if _, err := os.Stat(file); err == nil {
+		config.SerfConfig.KeyringFile = file
+	}
+	if err := loadKeyringFile(config.SerfConfig); err != nil {
+		return err
+	}
+	// Success!
+	return nil
+}
+
 // setupClient is used to setup the client if enabled
 func (a *Agent) setupClient() error {
 	if !a.config.Client.Enabled {

command/agent/agent_endpoint.go

@@ -3,7 +3,9 @@ package agent
 import (
 	"net"
 	"net/http"
+	"strings"
 
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/serf/serf"
 )
 
@@ -165,6 +167,56 @@ func (s *HTTPServer) updateServers(resp http.ResponseWriter, req *http.Request)
 	return nil, nil
 }
 
+// KeyringOperationRequest allows an operator to install/delete/use keys
+func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	srv := s.agent.Server()
+	if srv == nil {
+		return nil, CodedError(501, ErrInvalidMethod)
+	}
+
+	kmgr := srv.KeyManager()
+	var sresp *serf.KeyResponse
+	var err error
+
+	// Get the key from the req body
+	var args structs.KeyringRequest
+
+	//Get the op
+	op := strings.TrimPrefix(req.URL.Path, "/v1/agent/keyring/")
+
+	switch op {
+	case "list":
+		sresp, err = kmgr.ListKeys()
+	case "install":
+		if err := decodeBody(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.InstallKey(args.Key)
+	case "use":
+		if err := decodeBody(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.UseKey(args.Key)
+	case "remove":
+		if err := decodeBody(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.RemoveKey(args.Key)
+	default:
+		return nil, CodedError(404, "resource not found")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	kresp := structs.KeyringResponse{
+		Messages: sresp.Messages,
+		Keys:     sresp.Keys,
+		NumNodes: sresp.NumNodes,
+	}
+	return kresp, nil
+}
+
 type agentSelf struct {
 	Config *Config                      `json:"config"`
 	Member Member                       `json:"member,omitempty"`

command/agent/agent_endpoint_test.go

@@ -1,11 +1,15 @@
 package agent
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 func TestHTTP_AgentSelf(t *testing.T) {
@@ -177,3 +181,111 @@ func TestHTTP_AgentSetServers(t *testing.T) {
 		}
 	})
 }
+
+func TestHTTP_AgentListKeys(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 1 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}
+
+func TestHTTP_AgentInstallKey(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+
+		_, err = s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 2 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}
+
+func TestHTTP_AgentRemoveKey(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+		_, err = s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/remove", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+		if _, err = s.Server.KeyringOperationRequest(respW, req); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", nil)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 1 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}

command/agent/command.go

@@ -83,6 +83,7 @@ func (c *Command) readConfig() *Config {
 	flags.Var((*flaghelper.StringFlag)(&cmdConfig.Server.RetryJoin), "retry-join", "")
 	flags.IntVar(&cmdConfig.Server.RetryMaxAttempts, "retry-max", 0, "")
 	flags.StringVar(&cmdConfig.Server.RetryInterval, "retry-interval", "", "")
+	flags.StringVar(&cmdConfig.Server.EncryptKey, "encrypt", "", "gossip encryption key")
 
 	// Client-only options
 	flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "")
@@ -195,6 +196,17 @@ func (c *Command) readConfig() *Config {
 		return config
 	}
 
+	if config.Server.EncryptKey != "" {
+		if _, err := config.Server.EncryptBytes(); err != nil {
+			c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
+			return nil
+		}
+		keyfile := filepath.Join(config.DataDir, serfKeyring)
+		if _, err := os.Stat(keyfile); err == nil {
+			c.Ui.Error("WARNING: keyring exists but -encrypt given, using keyring")
+		}
+	}
+
 	// Parse the RetryInterval.
 	dur, err := time.ParseDuration(config.Server.RetryInterval)
 	if err != nil {
@@ -818,6 +830,9 @@ Server Options:
     bootstrapping the cluster. Once <num> servers have joined eachother,
     Nomad initiates the bootstrap process.
 
+  -encrypt=<key>
+    Provides the gossip encryption key
+
   -join=<address>
     Address of an agent to join at start time. Can be specified
     multiple times.

command/agent/config-test-fixtures/basic.hcl

@@ -68,6 +68,7 @@ server {
 	retry_max = 3
 	retry_interval = "15s"
 	rejoin_after_leave = true
+    encrypt = "abc"
 }
 telemetry {
 	statsite_address = "127.0.0.1:1234"

command/agent/config.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net"
@@ -244,6 +245,14 @@ type ServerConfig struct {
 	// the cluster until an explicit join is received. If this is set to
 	// true, we ignore the leave, and rejoin the cluster on start.
 	RejoinAfterLeave bool `mapstructure:"rejoin_after_leave"`
+
+	// Encryption key to use for the Serf communication
+	EncryptKey string `mapstructure:"encrypt" json:"-"`
+}
+
+// EncryptBytes returns the encryption key configured.
+func (s *ServerConfig) EncryptBytes() ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s.EncryptKey)
 }
 
 // Telemetry is the telemetry configuration for the server
@@ -669,6 +678,9 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.RejoinAfterLeave {
 		result.RejoinAfterLeave = true
 	}
+	if b.EncryptKey != "" {
+		result.EncryptKey = b.EncryptKey
+	}
 
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)

command/agent/config_parse.go

@@ -484,6 +484,7 @@ func parseServer(result **ServerConfig, list *ast.ObjectList) error {
 		"retry_max",
 		"retry_interval",
 		"rejoin_after_leave",
+		"encrypt",
 	}
 	if err := checkHCLKeys(listVal, valid); err != nil {
 		return err