DO NOT MERGE: implement support for connect native

api/services.go

@@ -138,7 +138,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
 
 // ConsulConnect represents a Consul Connect jobspec stanza.
 type ConsulConnect struct {
-	Native         bool
+	Native         string
 	SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"`
 	SidecarTask    *SidecarTask          `mapstructure:"sidecar_task"`
 }

client/allocrunner/taskrunner/connect_native_hook.go

@@ -0,0 +1,208 @@
+package taskrunner
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	hclog "github.com/hashicorp/go-hclog"
+	ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/pkg/errors"
+)
+
+const (
+	connectNativeHookName = "connect_native"
+)
+
+type connectNativeHookConfig struct {
+	consul consulTransportConfig
+	alloc  *structs.Allocation
+	logger hclog.Logger
+}
+
+func newConnectNativeHookConfig(alloc *structs.Allocation, consul *config.ConsulConfig, logger hclog.Logger) *connectNativeHookConfig {
+	return &connectNativeHookConfig{
+		alloc:  alloc,
+		logger: logger,
+		consul: newConsulTransportConfig(consul),
+	}
+}
+
+// connectNativeHook manages additional automagic configuration for a connect
+// native task.
+//
+// If nomad client is configured to talk to Consul using TLS (or other special
+// auth), the native task will inherit that configuration EXCEPT for the consul
+// token.
+//
+// If consul is configured with ACLs enabled, a Service Identity token will be
+// generated on behalf of the native service and supplied to the task.
+type connectNativeHook struct {
+	// alloc is the allocation with the connect native task being run
+	alloc *structs.Allocation
+
+	// consulConfig is used to enable the connect native enabled task to
+	// communicate with consul directly, as is necessary for the task to request
+	// its connect mTLS certificates.
+	consulConfig consulTransportConfig
+
+	// logger is used to log things
+	logger hclog.Logger
+}
+
+func newConnectNativeHook(c *connectNativeHookConfig) *connectNativeHook {
+	return &connectNativeHook{
+		alloc:        c.alloc,
+		consulConfig: c.consul,
+		logger:       c.logger.Named(connectNativeHookName),
+	}
+}
+
+func (connectNativeHook) Name() string {
+	return connectNativeHookName
+}
+
+func (h *connectNativeHook) Prestart(
+	ctx context.Context,
+	request *ifs.TaskPrestartRequest,
+	response *ifs.TaskPrestartResponse) error {
+
+	if !request.Task.Kind.IsConnectNative() {
+		response.Done = true
+		return nil
+	}
+
+	// copy TLS certificates
+	if err := h.copyCertificates(h.consulConfig, request.TaskDir.SecretsDir); err != nil {
+		h.logger.Error("failed to copy Consul TLS certificates", "error", err)
+		return err
+	}
+
+	// set environment variables for communicating with Consul agent
+	response.Env = h.tlsEnv()
+	if err := h.maybeSetSITokenEnv(request.TaskDir.SecretsDir, request.Task.Name, response.Env); err != nil {
+		h.logger.Error("failed to load Consul Service Identity Token", "error", err, "task", request.Task.Name)
+		return err
+	}
+	fmt.Println("response.Env:", response.Env)
+
+	// tls/acl setup for native task done
+	response.Done = true
+	return nil
+}
+
+const (
+	secretCAFilename       = "consul_ca_file"
+	secretCertfileFilename = "consul_cert_file"
+	secretKeyfileFilename  = "consul_key_file"
+)
+
+func (h *connectNativeHook) copyCertificates(consulConfig consulTransportConfig, dir string) error {
+	if err := h.copyCertificate(consulConfig.CAFile, dir, secretCAFilename); err != nil {
+		return err
+	}
+	if err := h.copyCertificate(consulConfig.CertFile, dir, secretCertfileFilename); err != nil {
+		return err
+	}
+	if err := h.copyCertificate(consulConfig.KeyFile, dir, secretKeyfileFilename); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (connectNativeHook) copyCertificate(source, dir, name string) error {
+	if source == "" {
+		return nil
+	}
+
+	original, err := os.Open(source)
+	if err != nil {
+		return errors.Wrap(err, "failed to open consul TLS certificate")
+	}
+	defer original.Close()
+
+	destination := filepath.Join(dir, name)
+	fd, err := os.Create(destination)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create secrets/%s", name)
+	}
+	defer fd.Close()
+
+	if _, err := io.Copy(fd, original); err != nil {
+		return errors.Wrapf(err, "failed to copy certificate secrets/%s", name)
+	}
+
+	if err := fd.Sync(); err != nil {
+		return errors.Wrapf(err, "failed to write secrets/%s", name)
+	}
+
+	return nil
+}
+
+// tlsEnv creates a set of additional of environment variables to be used when launching
+// the connect native task. This will enable the task to communicate with Consul
+// if Consul has transport security turned on.
+//
+// We do NOT set CONSUL_HTTP_TOKEN from the nomad agent's consul config, as that
+// is a separate security concern addressed by the service identity hook.
+func (h *connectNativeHook) tlsEnv() map[string]string {
+
+	// todo: maybeLoadSIToken references req.TaskDir.SecretsDir, can we do the same?
+	//  probably no - this is referenced by the the task, the envoy version only
+	//  needs the file in the context of the envoy bootstrap command, which runs
+	//  on the host
+
+	m := make(map[string]string)
+
+	if h.consulConfig.CAFile != "" {
+		m["CONSUL_CACERT"] = filepath.Join("/secrets", secretCAFilename)
+	}
+
+	if h.consulConfig.CertFile != "" {
+		m["CONSUL_CLIENT_CERT"] = filepath.Join("/secrets", secretCertfileFilename)
+	}
+
+	if h.consulConfig.KeyFile != "" {
+		m["CONSUL_CLIENT_KEY"] = filepath.Join("/secrets", secretKeyfileFilename)
+	}
+
+	if v := h.consulConfig.Auth; v != "" {
+		m["CONSUL_HTTP_AUTH"] = v
+	}
+	if v := h.consulConfig.SSL; v != "" {
+		m["CONSUL_HTTP_SSL"] = v
+	}
+	if v := h.consulConfig.VerifySSL; v != "" {
+		m["CONSUL_HTTP_SSL_VERIFY"] = v
+	}
+
+	return m
+}
+
+// maybeSetSITokenEnv will set the CONSUL_HTTP_TOKEN environment variable in
+// the given env map, if the token is found to exist in the task's secrets
+// directory.
+//
+// Following the pattern of the envoy_bootstrap_hook, the Consul Service Identity
+// ACL Token is generated prior to this hook, if Consul ACLs are enabled. This is
+// done in the sids_hook, which places the token at secrets/si_token in the task
+// workspace. The content of that file is the SI token specific to this task
+// instance.
+func (h *connectNativeHook) maybeSetSITokenEnv(dir, task string, env map[string]string) error {
+	token, err := ioutil.ReadFile(filepath.Join(dir, sidsTokenFile))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return errors.Wrapf(err, "failed to load SI token for native task %s", task)
+		}
+		h.logger.Trace("no SI token to load for native task", "task", task)
+		return nil // token file DNE; acls not enabled
+	}
+	h.logger.Trace("recovered pre-existing SI token for native task", "task", task)
+	env["CONSUL_HTTP_TOKEN"] = string(token)
+	return nil
+}

client/allocrunner/taskrunner/connect_native_hook_test.go

@@ -0,0 +1,60 @@
+package taskrunner
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConnectNativeHook_Name(t *testing.T) {
+	t.Parallel()
+	name := new(connectNativeHook).Name()
+	require.Equal(t, "connect_native", name)
+}
+
+func setupCertDirs(t *testing.T) (string, string) {
+	fd, err := ioutil.TempFile("", "connect_native_testcert")
+	require.NoError(t, err)
+	_, err = fd.WriteString("ABCDEF")
+	require.NoError(t, err)
+	err = fd.Close()
+	require.NoError(t, err)
+
+	d, err := ioutil.TempDir("", "connect_native_testsecrets")
+	require.NoError(t, err)
+	return fd.Name(), d
+}
+
+func cleanupCertDirs(t *testing.T, original, secrets string) {
+	err := os.Remove(original)
+	require.NoError(t, err)
+	err = os.RemoveAll(secrets)
+	require.NoError(t, err)
+}
+
+func TestConnectNativeHook_copyCertificate(t *testing.T) {
+	t.Parallel()
+
+	f, d := setupCertDirs(t)
+	defer cleanupCertDirs(t, f, d)
+
+	t.Run("no source", func(t *testing.T) {
+		err := new(connectNativeHook).copyCertificate("", d, "out.pem")
+		require.NoError(t, err)
+	})
+
+	t.Run("normal", func(t *testing.T) {
+		err := new(connectNativeHook).copyCertificate(f, d, "out.pem")
+		require.NoError(t, err)
+		b, err := ioutil.ReadFile(filepath.Join(d, "out.pem"))
+		require.NoError(t, err)
+		require.Equal(t, "ABCDEF", string(b))
+	})
+}
+
+func TestConnectNativeHook_copyCertificates(t *testing.T) {
+	t.Parallel()
+}

client/allocrunner/taskrunner/envoybootstrap_hook.go

@@ -22,7 +22,7 @@ import (
 
 const envoyBootstrapHookName = "envoy_bootstrap"
 
-type envoyBootstrapConsulConfig struct {
+type consulTransportConfig struct {
 	HTTPAddr  string // required
 	Auth      string // optional, env CONSUL_HTTP_AUTH
 	SSL       string // optional, env CONSUL_HTTP_SSL
@@ -33,8 +33,20 @@ type envoyBootstrapConsulConfig struct {
 	// CAPath (dir) not supported by Nomad's config object
 }
 
+func newConsulTransportConfig(consul *config.ConsulConfig) consulTransportConfig {
+	return consulTransportConfig{
+		HTTPAddr:  consul.Addr,
+		Auth:      consul.Auth,
+		SSL:       decodeTriState(consul.EnableSSL),
+		VerifySSL: decodeTriState(consul.VerifySSL),
+		CAFile:    consul.CAFile,
+		CertFile:  consul.CertFile,
+		KeyFile:   consul.KeyFile,
+	}
+}
+
 type envoyBootstrapHookConfig struct {
-	consul envoyBootstrapConsulConfig
+	consul consulTransportConfig
 	alloc  *structs.Allocation
 	logger hclog.Logger
 }
@@ -54,15 +66,7 @@ func newEnvoyBootstrapHookConfig(alloc *structs.Allocation, consul *config.Consu
 	return &envoyBootstrapHookConfig{
 		alloc:  alloc,
 		logger: logger,
-		consul: envoyBootstrapConsulConfig{
-			HTTPAddr:  consul.Addr,
-			Auth:      consul.Auth,
-			SSL:       decodeTriState(consul.EnableSSL),
-			VerifySSL: decodeTriState(consul.VerifySSL),
-			CAFile:    consul.CAFile,
-			CertFile:  consul.CertFile,
-			KeyFile:   consul.KeyFile,
-		},
+		consul: newConsulTransportConfig(consul),
 	}
 }
 
@@ -81,7 +85,7 @@ type envoyBootstrapHook struct {
 	// the bootstrap.json config. Runtime Envoy configuration is done via
 	// Consul's gRPC endpoint. There are many security parameters to configure
 	// before contacting Consul.
-	consulConfig envoyBootstrapConsulConfig
+	consulConfig consulTransportConfig
 
 	// logger is used to log things
 	logger hclog.Logger
@@ -269,7 +273,7 @@ func (h *envoyBootstrapHook) execute(cmd *exec.Cmd) (string, error) {
 // along to the exec invocation of consul which will then generate the bootstrap
 // configuration file for envoy.
 type envoyBootstrapArgs struct {
-	consulConfig   envoyBootstrapConsulConfig
+	consulConfig   consulTransportConfig
 	sidecarFor     string
 	grpcAddr       string
 	envoyAdminBind string

client/allocrunner/taskrunner/envoybootstrap_hook_test.go

@@ -93,11 +93,11 @@ func TestEnvoyBootstrapHook_decodeTriState(t *testing.T) {
 }
 
 var (
-	consulPlainConfig = envoyBootstrapConsulConfig{
+	consulPlainConfig = consulTransportConfig{
 		HTTPAddr: "2.2.2.2",
 	}
 
-	consulTLSConfig = envoyBootstrapConsulConfig{
+	consulTLSConfig = consulTransportConfig{
 		HTTPAddr:  "2.2.2.2",            // arg
 		Auth:      "user:password",      // env
 		SSL:       "true",               // env

client/allocrunner/taskrunner/task_runner_hooks.go

@@ -127,10 +127,16 @@ func (tr *TaskRunner) initHooks() {
 			}))
 		}
 
-		// envoy bootstrap must execute after sidsHook maybe sets SI token
-		tr.runnerHooks = append(tr.runnerHooks, newEnvoyBootstrapHook(
-			newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
-		))
+		if task.Kind.IsConnectProxy() {
+			// envoy bootstrap must execute after sidsHook maybe sets SI token
+			tr.runnerHooks = append(tr.runnerHooks, newEnvoyBootstrapHook(
+				newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger),
+			))
+		} else if task.Kind.IsConnectNative() {
+			tr.runnerHooks = append(tr.runnerHooks, newConnectNativeHook(
+				newConnectNativeHookConfig(alloc, tr.clientConfig.ConsulConfig, tr.logger),
+			))
+		}
 	}
 
 	// If there are any script checks, add the hook

command/agent/consul/connect.go

@@ -17,7 +17,7 @@ func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.
 		return nil, nil
 	}
 
-	if nc.Native {
+	if nc.IsNative() {
 		return &api.AgentServiceConnect{Native: true}, nil
 	}