Merge pull request #1614 from hashicorp/f-nomad-0.5

F nomad 0.5

.travis.yml

@@ -16,6 +16,7 @@ matrix:
 branches:
   only:
     - master
+    - /^f-.*$/
 
 before_install:  
     - sudo apt-get update

Vagrantfile

@@ -7,7 +7,6 @@ VAGRANTFILE_API_VERSION = "2"
 DEFAULT_CPU_COUNT = 2
 $script = <<SCRIPT
 GO_VERSION="1.7"
-CONSUL_VERSION="0.6.4"
 
 # Install Prereq Packages
 sudo apt-get update
@@ -44,14 +43,6 @@ sudo mv /tmp/gopath.sh /etc/profile.d/gopath.sh
 sudo chmod 0755 /etc/profile.d/gopath.sh
 source /etc/profile.d/gopath.sh
 
-echo Fetching Consul...
-cd /tmp/
-wget https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip -O consul.zip
-echo Installing Consul...
-unzip consul.zip
-sudo chmod +x consul
-sudo mv consul /usr/bin/consul
-
 # Install Docker
 echo deb https://apt.dockerproject.org/repo ubuntu-`lsb_release -c | awk '{print $2}'` main | sudo tee /etc/apt/sources.list.d/docker.list
 sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
@@ -67,8 +58,10 @@ sudo usermod -aG docker vagrant
 # Setup Nomad for development
 cd /opt/gopath/src/github.com/hashicorp/nomad && make bootstrap
 
-# Install rkt
+# Install rkt, consul and vault
 bash scripts/install_rkt.sh
+bash scripts/install_consul.sh
+bash scripts/install_vault.sh
 
 # CD into the nomad working directory when we login to the VM
 grep "cd /opt/gopath/src/github.com/hashicorp/nomad" ~/.profile || echo "cd /opt/gopath/src/github.com/hashicorp/nomad" >> ~/.profile

api/jobs.go

@@ -201,6 +201,7 @@ type Job struct {
 	Update            *UpdateStrategy
 	Periodic          *PeriodicConfig
 	Meta              map[string]string
+	VaultToken        string
 	Status            string
 	StatusDescription string
 	CreateIndex       uint64

api/tasks.go

@@ -140,6 +140,7 @@ type Task struct {
 	KillTimeout time.Duration
 	LogConfig   *LogConfig
 	Artifacts   []*TaskArtifact
+	Vault       *Vault
 }
 
 // TaskArtifact is used to download artifacts before running a task.
@@ -149,6 +150,10 @@ type TaskArtifact struct {
 	RelativeDest  string
 }
 
+type Vault struct {
+	Policies []string
+}
+
 // NewTask creates and initializes a new Task.
 func NewTask(name, driver string) *Task {
 	return &Task{

client/client_test.go

@@ -38,6 +38,7 @@ func getPort() int {
 func testServer(t *testing.T, cb func(*nomad.Config)) (*nomad.Server, string) {
 	// Setup the default settings
 	config := nomad.DefaultConfig()
+	config.VaultConfig.Enabled = false
 	config.Build = "unittest"
 	config.DevMode = true
 	config.RPCAddr = &net.TCPAddr{

client/config/config.go

@@ -118,6 +118,9 @@ type Config struct {
 	// ConsulConfig is this Agent's Consul configuration
 	ConsulConfig *config.ConsulConfig
 
+	// VaultConfig is this Agent's Vault configuration
+	VaultConfig *config.VaultConfig
+
 	// StatsCollectionInterval is the interval at which the Nomad client
 	// collects resource usage stats
 	StatsCollectionInterval time.Duration
@@ -137,6 +140,9 @@ func (c *Config) Copy() *Config {
 	nc.Node = nc.Node.Copy()
 	nc.Servers = structs.CopySliceString(nc.Servers)
 	nc.Options = structs.CopyMapStringString(nc.Options)
+	nc.GloballyReservedPorts = structs.CopySliceInt(c.GloballyReservedPorts)
+	nc.ConsulConfig = c.ConsulConfig.Copy()
+	nc.VaultConfig = c.VaultConfig.Copy()
 	return nc
 }
 

command/agent/agent.go

@@ -239,7 +239,9 @@ func (a *Agent) serverConfig() (*nomad.Config, error) {
 		return nil, fmt.Errorf("server_service_name must be set when auto_advertise is enabled")
 	}
 
+	// Add the Consul and Vault configs
 	conf.ConsulConfig = a.config.Consul
+	conf.VaultConfig = a.config.Vault
 
 	return conf, nil
 }
@@ -350,6 +352,7 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 	}
 
 	conf.ConsulConfig = a.config.Consul
+	conf.VaultConfig = a.config.Vault
 	conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval
 	conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics
 	conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics

command/agent/config-test-fixtures/basic.hcl

@@ -107,3 +107,16 @@ consul {
     client_auto_join = false
     auto_advertise = false
 }
+vault {
+    address = "127.0.0.1:9500"
+    allow_unauthenticated = true
+    task_token_ttl = "1s"
+    enabled = false
+    token = "12345"
+    tls_ca_file = "/path/to/ca/file"
+    tls_ca_path = "/path/to/ca"
+    tls_cert_file = "/path/to/cert/file"
+    tls_key_file = "/path/to/key/file"
+    tls_server_name = "foobar"
+    tls_skip_verify = true
+}

command/agent/config.go

@@ -88,6 +88,10 @@ type Config struct {
 	// discover the current Nomad servers.
 	Consul *config.ConsulConfig `mapstructure:"consul"`
 
+	// Vault contains the configuration for the Vault Agent and
+	// parameters necessary to derive tokens.
+	Vault *config.VaultConfig `mapstructure:"vault"`
+
 	// NomadConfig is used to override the default config.
 	// This is largly used for testing purposes.
 	NomadConfig *nomad.Config `mapstructure:"-" json:"-"`
@@ -448,6 +452,7 @@ func DefaultConfig() *Config {
 		AdvertiseAddrs: &AdvertiseAddrs{},
 		Atlas:          &AtlasConfig{},
 		Consul:         config.DefaultConsulConfig(),
+		Vault:          config.DefaultVaultConfig(),
 		Client: &ClientConfig{
 			Enabled:        false,
 			NetworkSpeed:   100,
@@ -604,6 +609,14 @@ func (c *Config) Merge(b *Config) *Config {
 		result.Consul = result.Consul.Merge(b.Consul)
 	}
 
+	// Apply the Vault Configuration
+	if result.Vault == nil && b.Vault != nil {
+		vaultConfig := *b.Vault
+		result.Vault = &vaultConfig
+	} else if b.Vault != nil {
+		result.Vault = result.Vault.Merge(b.Vault)
+	}
+
 	// Merge config files lists
 	result.Files = append(result.Files, b.Files...)
 

command/agent/config_parse.go

@@ -93,6 +93,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 		"disable_anonymous_signature",
 		"atlas",
 		"consul",
+		"vault",
 		"http_api_response_headers",
 	}
 	if err := checkHCLKeys(list, valid); err != nil {
@@ -113,6 +114,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 	delete(m, "telemetry")
 	delete(m, "atlas")
 	delete(m, "consul")
+	delete(m, "vault")
 	delete(m, "http_api_response_headers")
 
 	// Decode the rest
@@ -176,6 +178,13 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 		}
 	}
 
+	// Parse the vault config
+	if o := list.Filter("vault"); len(o.Items) > 0 {
+		if err := parseVaultConfig(&result.Vault, o); err != nil {
+			return multierror.Prefix(err, "vault ->")
+		}
+	}
+
 	// Parse out http_api_response_headers fields. These are in HCL as a list so
 	// we need to iterate over them and merge them.
 	if headersO := list.Filter("http_api_response_headers"); len(headersO.Items) > 0 {
@@ -633,6 +642,56 @@ func parseConsulConfig(result **config.ConsulConfig, list *ast.ObjectList) error
 	return nil
 }
 
+func parseVaultConfig(result **config.VaultConfig, list *ast.ObjectList) error {
+	list = list.Elem()
+	if len(list.Items) > 1 {
+		return fmt.Errorf("only one 'vault' block allowed")
+	}
+
+	// Get our Vault object
+	listVal := list.Items[0].Val
+
+	// Check for invalid keys
+	valid := []string{
+		"address",
+		"allow_unauthenticated",
+		"enabled",
+		"task_token_ttl",
+		"tls_ca_file",
+		"tls_ca_path",
+		"tls_cert_file",
+		"tls_key_file",
+		"tls_server_name",
+		"tls_skip_verify",
+		"token",
+	}
+
+	if err := checkHCLKeys(listVal, valid); err != nil {
+		return err
+	}
+
+	var m map[string]interface{}
+	if err := hcl.DecodeObject(&m, listVal); err != nil {
+		return err
+	}
+
+	vaultConfig := config.DefaultVaultConfig()
+	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
+		WeaklyTypedInput: true,
+		Result:           &vaultConfig,
+	})
+	if err != nil {
+		return err
+	}
+	if err := dec.Decode(m); err != nil {
+		return err
+	}
+
+	*result = vaultConfig
+	return nil
+}
+
 func checkHCLKeys(node ast.Node, valid []string) error {
 	var list *ast.ObjectList
 	switch n := node.(type) {

command/agent/config_parse_test.go

@@ -122,6 +122,19 @@ func TestConfig_Parse(t *testing.T) {
 					ClientAutoJoin:    false,
 					AutoAdvertise:     false,
 				},
+				Vault: &config.VaultConfig{
+					Addr:                 "127.0.0.1:9500",
+					AllowUnauthenticated: true,
+					Enabled:              false,
+					TLSCaFile:            "/path/to/ca/file",
+					TLSCaPath:            "/path/to/ca",
+					TLSCertFile:          "/path/to/cert/file",
+					TLSKeyFile:           "/path/to/key/file",
+					TLSServerName:        "foobar",
+					TLSSkipVerify:        true,
+					TaskTokenTTL:         "1s",
+					Token:                "12345",
+				},
 				HTTPAPIResponseHeaders: map[string]string{
 					"Access-Control-Allow-Origin": "*",
 				},

command/agent/config_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/nomad/structs/config"
 )
 
 func TestConfig_Merge(t *testing.T) {
@@ -94,6 +95,34 @@ func TestConfig_Merge(t *testing.T) {
 		HTTPAPIResponseHeaders: map[string]string{
 			"Access-Control-Allow-Origin": "*",
 		},
+		Vault: &config.VaultConfig{
+			Token:                "1",
+			AllowUnauthenticated: false,
+			TaskTokenTTL:         "1",
+			Addr:                 "1",
+			TLSCaFile:            "1",
+			TLSCaPath:            "1",
+			TLSCertFile:          "1",
+			TLSKeyFile:           "1",
+			TLSSkipVerify:        false,
+			TLSServerName:        "1",
+		},
+		Consul: &config.ConsulConfig{
+			ServerServiceName: "1",
+			ClientServiceName: "1",
+			AutoAdvertise:     false,
+			Addr:              "1",
+			Timeout:           1 * time.Second,
+			Token:             "1",
+			Auth:              "1",
+			EnableSSL:         false,
+			VerifySSL:         false,
+			CAFile:            "1",
+			CertFile:          "1",
+			KeyFile:           "1",
+			ServerAutoJoin:    false,
+			ClientAutoJoin:    false,
+		},
 	}
 
 	c2 := &Config{
@@ -192,6 +221,34 @@ func TestConfig_Merge(t *testing.T) {
 			"Access-Control-Allow-Origin":  "*",
 			"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
 		},
+		Vault: &config.VaultConfig{
+			Token:                "2",
+			AllowUnauthenticated: true,
+			TaskTokenTTL:         "2",
+			Addr:                 "2",
+			TLSCaFile:            "2",
+			TLSCaPath:            "2",
+			TLSCertFile:          "2",
+			TLSKeyFile:           "2",
+			TLSSkipVerify:        true,
+			TLSServerName:        "2",
+		},
+		Consul: &config.ConsulConfig{
+			ServerServiceName: "2",
+			ClientServiceName: "2",
+			AutoAdvertise:     true,
+			Addr:              "2",
+			Timeout:           2 * time.Second,
+			Token:             "2",
+			Auth:              "2",
+			EnableSSL:         true,
+			VerifySSL:         true,
+			CAFile:            "2",
+			CertFile:          "2",
+			KeyFile:           "2",
+			ServerAutoJoin:    true,
+			ClientAutoJoin:    true,
+		},
 	}
 
 	result := c1.Merge(c2)