diff --git a/api/services.go b/api/services.go
index 880ba7f7973..34440124c7a 100644
--- a/api/services.go
+++ b/api/services.go
@@ -281,17 +281,77 @@ func (cp *ConsulProxy) Canonicalize() {
cp.Upstreams = nil
}
+ for _, upstream := range cp.Upstreams {
+ upstream.Canonicalize()
+ }
+
if len(cp.Config) == 0 {
cp.Config = nil
}
}
+// ConsulMeshGateway is used to configure mesh gateway usage when connecting to
+// a connect upstream in another datacenter.
+type ConsulMeshGateway struct {
+ // Mode configures how an upstream should be accessed with regard to using
+ // mesh gateways.
+ //
+ // local - the connect proxy makes outbound connections through mesh gateway
+ // originating in the same datacenter.
+ //
+ // remote - the connect proxy makes outbound connections to a mesh gateway
+ // in the destination datacenter.
+ //
+ // none (default) - no mesh gateway is used, the proxy makes outbound connections
+ // directly to destination services.
+ //
+ // https://www.consul.io/docs/connect/gateways/mesh-gateway#modes-of-operation
+ Mode string `mapstructure:"mode" hcl:"mode,optional"`
+}
+
+func (c *ConsulMeshGateway) Canonicalize() {
+ // Mode may be empty string, indicating behavior will defer to Consul
+ // service-defaults config entry.
+ return
+}
+
+func (c *ConsulMeshGateway) Copy() *ConsulMeshGateway {
+ if c == nil {
+ return nil
+ }
+
+ return &ConsulMeshGateway{
+ Mode: c.Mode,
+ }
+}
+
// ConsulUpstream represents a Consul Connect upstream jobspec stanza.
type ConsulUpstream struct {
- DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"`
- LocalBindPort int `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"`
- Datacenter string `mapstructure:"datacenter" hcl:"datacenter,optional"`
- LocalBindAddress string `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"`
+ DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"`
+ LocalBindPort int `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"`
+ Datacenter string `mapstructure:"datacenter" hcl:"datacenter,optional"`
+ LocalBindAddress string `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"`
+ MeshGateway *ConsulMeshGateway `mapstructure:"mesh_gateway" hcl:"mesh_gateway,block"`
+}
+
+func (cu *ConsulUpstream) Copy() *ConsulUpstream {
+ if cu == nil {
+ return nil
+ }
+ return &ConsulUpstream{
+ DestinationName: cu.DestinationName,
+ LocalBindPort: cu.LocalBindPort,
+ Datacenter: cu.Datacenter,
+ LocalBindAddress: cu.LocalBindAddress,
+ MeshGateway: cu.MeshGateway.Copy(),
+ }
+}
+
+func (cu *ConsulUpstream) Canonicalize() {
+ if cu == nil {
+ return
+ }
+ cu.MeshGateway.Canonicalize()
}
type ConsulExposeConfig struct {
@@ -326,8 +386,8 @@ type ConsulGateway struct {
// Terminating represents the Consul Configuration Entry for a Terminating Gateway.
Terminating *ConsulTerminatingConfigEntry `hcl:"terminating,block"`
- // Mesh is not yet supported.
- // Mesh *ConsulMeshConfigEntry
+ // Mesh indicates the Consul service should be a Mesh Gateway.
+ Mesh *ConsulMeshConfigEntry `hcl:"mesh,block"`
}
func (g *ConsulGateway) Canonicalize() {
@@ -643,6 +703,21 @@ func (e *ConsulTerminatingConfigEntry) Copy() *ConsulTerminatingConfigEntry {
}
}
-// ConsulMeshConfigEntry is not yet supported.
-// type ConsulMeshConfigEntry struct {
-// }
+// ConsulMeshConfigEntry is a stub used to represent that the gateway service type
+// should be for a Mesh Gateway. Unlike Ingress and Terminating, there is no
+// actual Consul Config Entry type for mesh-gateway, at least for now. We still
+// create a type for future proofing, instead just using a bool for example.
+type ConsulMeshConfigEntry struct {
+ // nothing in here
+}
+
+func (e *ConsulMeshConfigEntry) Canonicalize() {
+ return
+}
+
+func (e *ConsulMeshConfigEntry) Copy() *ConsulMeshConfigEntry {
+ if e == nil {
+ return nil
+ }
+ return new(ConsulMeshConfigEntry)
+}
diff --git a/api/services_test.go b/api/services_test.go
index 705787ef2ef..648c975d80c 100644
--- a/api/services_test.go
+++ b/api/services_test.go
@@ -216,6 +216,56 @@ func TestService_Connect_ConsulProxy_Canonicalize(t *testing.T) {
})
}
+func TestService_Connect_ConsulUpstream_Copy(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil upstream", func(t *testing.T) {
+ cu := (*ConsulUpstream)(nil)
+ result := cu.Copy()
+ require.Nil(t, result)
+ })
+
+ t.Run("complete upstream", func(t *testing.T) {
+ cu := &ConsulUpstream{
+ DestinationName: "dest1",
+ Datacenter: "dc2",
+ LocalBindPort: 2000,
+ LocalBindAddress: "10.0.0.1",
+ MeshGateway: &ConsulMeshGateway{Mode: "remote"},
+ }
+ result := cu.Copy()
+ require.Equal(t, cu, result)
+ })
+}
+
+func TestService_Connect_ConsulUpstream_Canonicalize(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil upstream", func(t *testing.T) {
+ cu := (*ConsulUpstream)(nil)
+ cu.Canonicalize()
+ require.Nil(t, cu)
+ })
+
+ t.Run("complete", func(t *testing.T) {
+ cu := &ConsulUpstream{
+ DestinationName: "dest1",
+ Datacenter: "dc2",
+ LocalBindPort: 2000,
+ LocalBindAddress: "10.0.0.1",
+ MeshGateway: &ConsulMeshGateway{Mode: ""},
+ }
+ cu.Canonicalize()
+ require.Equal(t, &ConsulUpstream{
+ DestinationName: "dest1",
+ Datacenter: "dc2",
+ LocalBindPort: 2000,
+ LocalBindAddress: "10.0.0.1",
+ MeshGateway: &ConsulMeshGateway{Mode: ""},
+ }, cu)
+ })
+}
+
func TestService_Connect_proxy_settings(t *testing.T) {
t.Parallel()
@@ -508,3 +558,75 @@ func TestService_ConsulTerminatingConfigEntry_Copy(t *testing.T) {
require.Equal(t, entry, result)
})
}
+
+func TestService_ConsulMeshConfigEntry_Canonicalize(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ ce := (*ConsulMeshConfigEntry)(nil)
+ ce.Canonicalize()
+ require.Nil(t, ce)
+ })
+
+ t.Run("instantiated", func(t *testing.T) {
+ ce := new(ConsulMeshConfigEntry)
+ ce.Canonicalize()
+ require.NotNil(t, ce)
+ })
+}
+
+func TestService_ConsulMeshConfigEntry_Copy(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ ce := (*ConsulMeshConfigEntry)(nil)
+ ce2 := ce.Copy()
+ require.Nil(t, ce2)
+ })
+
+ t.Run("instantiated", func(t *testing.T) {
+ ce := new(ConsulMeshConfigEntry)
+ ce2 := ce.Copy()
+ require.NotNil(t, ce2)
+ })
+}
+
+func TestService_ConsulMeshGateway_Canonicalize(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ c := (*ConsulMeshGateway)(nil)
+ c.Canonicalize()
+ require.Nil(t, c)
+ })
+
+ t.Run("unset mode", func(t *testing.T) {
+ c := &ConsulMeshGateway{Mode: ""}
+ c.Canonicalize()
+ require.Equal(t, "", c.Mode)
+ })
+
+ t.Run("set mode", func(t *testing.T) {
+ c := &ConsulMeshGateway{Mode: "remote"}
+ c.Canonicalize()
+ require.Equal(t, "remote", c.Mode)
+ })
+}
+
+func TestService_ConsulMeshGateway_Copy(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ c := (*ConsulMeshGateway)(nil)
+ result := c.Copy()
+ require.Nil(t, result)
+ })
+
+ t.Run("instantiated", func(t *testing.T) {
+ c := &ConsulMeshGateway{
+ Mode: "local",
+ }
+ result := c.Copy()
+ require.Equal(t, c, result)
+ })
+}
diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go
index d2ef3df53d1..ee58b03a0bd 100644
--- a/client/allocrunner/taskrunner/envoy_bootstrap_hook.go
+++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook.go
@@ -164,8 +164,18 @@ func (envoyBootstrapHook) Name() string {
}
func isConnectKind(kind string) bool {
- kinds := []string{structs.ConnectProxyPrefix, structs.ConnectIngressPrefix, structs.ConnectTerminatingPrefix}
- return helper.SliceStringContains(kinds, kind)
+ switch kind {
+ case structs.ConnectProxyPrefix:
+ return true
+ case structs.ConnectIngressPrefix:
+ return true
+ case structs.ConnectTerminatingPrefix:
+ return true
+ case structs.ConnectMeshPrefix:
+ return true
+ default:
+ return false
+ }
}
func (_ *envoyBootstrapHook) extractNameAndKind(kind structs.TaskKind) (string, string, error) {
@@ -183,7 +193,7 @@ func (_ *envoyBootstrapHook) extractNameAndKind(kind structs.TaskKind) (string,
return serviceKind, serviceName, nil
}
-func (h *envoyBootstrapHook) lookupService(svcKind, svcName, tgName string, taskEnv *taskenv.TaskEnv) (*structs.Service, error) {
+func (h *envoyBootstrapHook) lookupService(svcKind, svcName string, taskEnv *taskenv.TaskEnv) (*structs.Service, error) {
tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
interpolatedServices := taskenv.InterpolateServices(taskEnv, tg.Services)
@@ -221,7 +231,7 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart
return err
}
- service, err := h.lookupService(serviceKind, serviceName, h.alloc.TaskGroup, req.TaskEnv)
+ service, err := h.lookupService(serviceKind, serviceName, req.TaskEnv)
if err != nil {
return err
}
@@ -404,6 +414,11 @@ func (h *envoyBootstrapHook) proxyServiceID(group string, service *structs.Servi
return agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+group, service)
}
+// newEnvoyBootstrapArgs is used to prepare for the invocation of the
+// 'consul connect envoy' command with arguments which will bootstrap the connect
+// proxy or gateway.
+//
+// https://www.consul.io/commands/connect/envoy#consul-connect-envoy
func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
group string, service *structs.Service,
grpcAddr, envoyAdminBind, siToken, filepath string,
@@ -416,19 +431,23 @@ func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
)
namespace = h.getConsulNamespace()
+ id := h.proxyServiceID(group, service)
switch {
case service.Connect.HasSidecar():
- sidecarForID = h.proxyServiceID(group, service)
+ sidecarForID = id
case service.Connect.IsIngress():
- proxyID = h.proxyServiceID(group, service)
+ proxyID = id
gateway = "ingress"
case service.Connect.IsTerminating():
- proxyID = h.proxyServiceID(group, service)
+ proxyID = id
gateway = "terminating"
+ case service.Connect.IsMesh():
+ proxyID = id
+ gateway = "mesh"
}
- h.logger.Debug("bootstrapping envoy",
+ h.logger.Info("bootstrapping envoy",
"sidecar_for", service.Name, "bootstrap_file", filepath,
"sidecar_for_id", sidecarForID, "grpc_addr", grpcAddr,
"admin_bind", envoyAdminBind, "gateway", gateway,
diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go
index 5f9d78dfd7a..58e0ed988dd 100644
--- a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go
+++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go
@@ -193,6 +193,25 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
}, result)
})
+
+ t.Run("mesh gateway", func(t *testing.T) {
+ ebArgs := envoyBootstrapArgs{
+ consulConfig: consulPlainConfig,
+ grpcAddr: "1.1.1.1",
+ envoyAdminBind: "localhost:3333",
+ gateway: "my-mesh-gateway",
+ proxyID: "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-mesh-mesh-8080",
+ }
+ result := ebArgs.args()
+ require.Equal(t, []string{"connect", "envoy",
+ "-grpc-addr", "1.1.1.1",
+ "-http-addr", "2.2.2.2",
+ "-admin-bind", "localhost:3333",
+ "-bootstrap",
+ "-gateway", "my-mesh-gateway",
+ "-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-mesh-mesh-8080",
+ }, result)
+ })
}
func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
@@ -809,3 +828,12 @@ func TestTaskRunner_EnvoyBootstrapHook_grpcAddress(t *testing.T) {
require.Equal(t, "127.0.0.1:8502", hostH.grpcAddress(nil))
})
}
+
+func TestTaskRunner_EnvoyBootstrapHook_isConnectKind(t *testing.T) {
+ require.True(t, isConnectKind(structs.ConnectProxyPrefix))
+ require.True(t, isConnectKind(structs.ConnectIngressPrefix))
+ require.True(t, isConnectKind(structs.ConnectTerminatingPrefix))
+ require.True(t, isConnectKind(structs.ConnectMeshPrefix))
+ require.False(t, isConnectKind(""))
+ require.False(t, isConnectKind("something"))
+}
diff --git a/command/agent/consul/connect.go b/command/agent/consul/connect.go
index 267f26aa77f..dfc7b6c1ba6 100644
--- a/command/agent/consul/connect.go
+++ b/command/agent/consul/connect.go
@@ -198,11 +198,36 @@ func connectUpstreams(in []structs.ConsulUpstream) []api.Upstream {
LocalBindPort: upstream.LocalBindPort,
Datacenter: upstream.Datacenter,
LocalBindAddress: upstream.LocalBindAddress,
+ MeshGateway: connectMeshGateway(upstream.MeshGateway),
}
}
return upstreams
}
+// connectMeshGateway creates an api.MeshGatewayConfig from the nomad upstream
+// block. A non-existent config or unsupported gateway mode will default to the
+// Consul default mode.
+func connectMeshGateway(in *structs.ConsulMeshGateway) api.MeshGatewayConfig {
+ gw := api.MeshGatewayConfig{
+ Mode: api.MeshGatewayModeDefault,
+ }
+
+ if in == nil {
+ return gw
+ }
+
+ switch in.Mode {
+ case "local":
+ gw.Mode = api.MeshGatewayModeLocal
+ case "remote":
+ gw.Mode = api.MeshGatewayModeRemote
+ case "none":
+ gw.Mode = api.MeshGatewayModeNone
+ }
+
+ return gw
+}
+
func connectProxyConfig(cfg map[string]interface{}, port int) map[string]interface{} {
if cfg == nil {
cfg = make(map[string]interface{})
diff --git a/command/agent/consul/connect_test.go b/command/agent/consul/connect_test.go
index 52f2eea807b..4f34306642b 100644
--- a/command/agent/consul/connect_test.go
+++ b/command/agent/consul/connect_test.go
@@ -514,7 +514,7 @@ func TestConnect_newConnectGateway(t *testing.T) {
ConnectTimeout: helper.TimeToPtr(1 * time.Second),
EnvoyGatewayBindTaggedAddresses: true,
EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
- "service1": &structs.ConsulGatewayBindAddress{
+ "service1": {
Address: "10.0.0.1",
Port: 2000,
},
@@ -532,7 +532,7 @@ func TestConnect_newConnectGateway(t *testing.T) {
"connect_timeout_ms": int64(1000),
"envoy_gateway_bind_tagged_addresses": true,
"envoy_gateway_bind_addresses": map[string]*structs.ConsulGatewayBindAddress{
- "service1": &structs.ConsulGatewayBindAddress{
+ "service1": {
Address: "10.0.0.1",
Port: 2000,
},
@@ -544,3 +544,32 @@ func TestConnect_newConnectGateway(t *testing.T) {
}, result)
})
}
+
+func Test_connectMeshGateway(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ result := connectMeshGateway(nil)
+ require.Equal(t, api.MeshGatewayConfig{Mode: api.MeshGatewayModeDefault}, result)
+ })
+
+ t.Run("local", func(t *testing.T) {
+ result := connectMeshGateway(&structs.ConsulMeshGateway{Mode: "local"})
+ require.Equal(t, api.MeshGatewayConfig{Mode: api.MeshGatewayModeLocal}, result)
+ })
+
+ t.Run("remote", func(t *testing.T) {
+ result := connectMeshGateway(&structs.ConsulMeshGateway{Mode: "remote"})
+ require.Equal(t, api.MeshGatewayConfig{Mode: api.MeshGatewayModeRemote}, result)
+ })
+
+ t.Run("none", func(t *testing.T) {
+ result := connectMeshGateway(&structs.ConsulMeshGateway{Mode: "none"})
+ require.Equal(t, api.MeshGatewayConfig{Mode: api.MeshGatewayModeNone}, result)
+ })
+
+ t.Run("nonsense", func(t *testing.T) {
+ result := connectMeshGateway(nil)
+ require.Equal(t, api.MeshGatewayConfig{Mode: api.MeshGatewayModeDefault}, result)
+ })
+}
diff --git a/command/agent/consul/service_client.go b/command/agent/consul/service_client.go
index 8bd43483836..acddb826ec8 100644
--- a/command/agent/consul/service_client.go
+++ b/command/agent/consul/service_client.go
@@ -14,6 +14,7 @@ import (
"github.com/armon/go-metrics"
log "github.com/hashicorp/go-hclog"
+ "github.com/hashicorp/nomad/helper/envoy"
"github.com/pkg/errors"
"github.com/hashicorp/consul/api"
@@ -961,11 +962,26 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w
kind = api.ServiceKindTerminatingGateway
// set the default port if bridge / default listener set
if defaultBind, exists := service.Connect.Gateway.Proxy.EnvoyGatewayBindAddresses["default"]; exists {
- portLabel := fmt.Sprintf("%s-%s", structs.ConnectTerminatingPrefix, service.Name)
+ portLabel := envoy.PortLabel(structs.ConnectTerminatingPrefix, service.Name, "")
if dynPort, ok := workload.Ports.Get(portLabel); ok {
defaultBind.Port = dynPort.Value
}
}
+ case service.Connect.IsMesh():
+ kind = api.ServiceKindMeshGateway
+ // wan uses the service port label, which is typically on a discrete host_network
+ if wanBind, exists := service.Connect.Gateway.Proxy.EnvoyGatewayBindAddresses["wan"]; exists {
+ if wanPort, ok := workload.Ports.Get(service.PortLabel); ok {
+ wanBind.Port = wanPort.Value
+ }
+ }
+ // lan uses a nomad generated dynamic port on the default network
+ if lanBind, exists := service.Connect.Gateway.Proxy.EnvoyGatewayBindAddresses["lan"]; exists {
+ portLabel := envoy.PortLabel(structs.ConnectMeshPrefix, service.Name, "lan")
+ if dynPort, ok := workload.Ports.Get(portLabel); ok {
+ lanBind.Port = dynPort.Value
+ }
+ }
}
// Build the Consul Service registration request
diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go
index 844802ee232..507a4c95521 100644
--- a/command/agent/job_endpoint.go
+++ b/command/agent/job_endpoint.go
@@ -7,7 +7,7 @@ import (
"strings"
"github.com/golang/snappy"
- "github.com/hashicorp/nomad/api"
+ api "github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/jobspec"
"github.com/hashicorp/nomad/jobspec2"
@@ -1362,6 +1362,7 @@ func apiConnectGatewayToStructs(in *api.ConsulGateway) *structs.ConsulGateway {
Proxy: apiConnectGatewayProxyToStructs(in.Proxy),
Ingress: apiConnectIngressGatewayToStructs(in.Ingress),
Terminating: apiConnectTerminatingGatewayToStructs(in.Terminating),
+ Mesh: apiConnectMeshGatewayToStructs(in.Mesh),
}
}
@@ -1494,6 +1495,13 @@ func apiConnectTerminatingServiceToStructs(in *api.ConsulLinkedService) *structs
}
}
+func apiConnectMeshGatewayToStructs(in *api.ConsulMeshConfigEntry) *structs.ConsulMeshConfigEntry {
+ if in == nil {
+ return nil
+ }
+ return new(structs.ConsulMeshConfigEntry)
+}
+
func apiConnectSidecarServiceToStructs(in *api.ConsulSidecarService) *structs.ConsulSidecarService {
if in == nil {
return nil
@@ -1530,11 +1538,21 @@ func apiUpstreamsToStructs(in []*api.ConsulUpstream) []structs.ConsulUpstream {
LocalBindPort: upstream.LocalBindPort,
Datacenter: upstream.Datacenter,
LocalBindAddress: upstream.LocalBindAddress,
+ MeshGateway: apiMeshGatewayToStructs(upstream.MeshGateway),
}
}
return upstreams
}
+func apiMeshGatewayToStructs(in *api.ConsulMeshGateway) *structs.ConsulMeshGateway {
+ if in == nil {
+ return nil
+ }
+ return &structs.ConsulMeshGateway{
+ Mode: in.Mode,
+ }
+}
+
func apiConsulExposeConfigToStructs(in *api.ConsulExposeConfig) *structs.ConsulExposeConfig {
if in == nil {
return nil
diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go
index 4d633bac69f..648fa620a71 100644
--- a/command/agent/job_endpoint_test.go
+++ b/command/agent/job_endpoint_test.go
@@ -9,7 +9,7 @@ import (
"time"
"github.com/golang/snappy"
- "github.com/hashicorp/nomad/api"
+ api "github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
@@ -3138,14 +3138,23 @@ func TestConversion_apiUpstreamsToStructs(t *testing.T) {
LocalBindPort: 8000,
Datacenter: "dc2",
LocalBindAddress: "127.0.0.2",
+ MeshGateway: &structs.ConsulMeshGateway{Mode: "local"},
}}, apiUpstreamsToStructs([]*api.ConsulUpstream{{
DestinationName: "upstream",
LocalBindPort: 8000,
Datacenter: "dc2",
LocalBindAddress: "127.0.0.2",
+ MeshGateway: &api.ConsulMeshGateway{Mode: "local"},
}}))
}
+func TestConversion_apiConsulMeshGatewayToStructs(t *testing.T) {
+ t.Parallel()
+ require.Nil(t, apiMeshGatewayToStructs(nil))
+ require.Equal(t, &structs.ConsulMeshGateway{Mode: "remote"},
+ apiMeshGatewayToStructs(&api.ConsulMeshGateway{Mode: "remote"}))
+}
+
func TestConversion_apiConnectSidecarServiceProxyToStructs(t *testing.T) {
t.Parallel()
require.Nil(t, apiConnectSidecarServiceProxyToStructs(nil))
@@ -3313,6 +3322,22 @@ func TestConversion_ApiConsulConnectToStructs(t *testing.T) {
}))
})
+ t.Run("gateway mesh", func(t *testing.T) {
+ require.Equal(t, &structs.ConsulConnect{
+ Gateway: &structs.ConsulGateway{
+ Mesh: &structs.ConsulMeshConfigEntry{
+ // nothing
+ },
+ },
+ }, ApiConsulConnectToStructs(&api.ConsulConnect{
+ Gateway: &api.ConsulGateway{
+ Mesh: &api.ConsulMeshConfigEntry{
+ // nothing
+ },
+ },
+ }))
+ })
+
t.Run("native", func(t *testing.T) {
require.Equal(t, &structs.ConsulConnect{
Native: true,
diff --git a/helper/envoy/envoy.go b/helper/envoy/envoy.go
index 2915ee549b1..e07e1ccc41a 100644
--- a/helper/envoy/envoy.go
+++ b/helper/envoy/envoy.go
@@ -2,6 +2,10 @@
// selecting an envoy version.
package envoy
+import (
+ "fmt"
+)
+
const (
// SidecarMetaParam is the parameter name used to configure the connect sidecar
// at the client level. Setting this option in client configuration takes the
@@ -27,7 +31,7 @@ const (
// gateway proxies, when they are injected in the job connect mutator.
GatewayConfigVar = "${meta." + GatewayMetaParam + "}"
- // envoyImageFormat is the default format string used for official envoy Docker
+ // ImageFormat is the default format string used for official envoy Docker
// images with the tag being the semver of the version of envoy. Nomad fakes
// interpolation of ${NOMAD_envoy_version} by replacing it with the version
// string for envoy that Consul reports as preferred.
@@ -37,7 +41,7 @@ const (
// to something like: "custom/envoy:${NOMAD_envoy_version}".
ImageFormat = "envoyproxy/envoy:v" + VersionVar
- // envoyVersionVar will be replaced with the Envoy version string when
+ // VersionVar will be replaced with the Envoy version string when
// used in the meta.connect.sidecar_image variable.
VersionVar = "${NOMAD_envoy_version}"
@@ -47,3 +51,13 @@ const (
// dynamic envoy versions.
FallbackImage = "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09"
)
+
+// PortLabel creates a consistent port label using the inputs of a prefix,
+// service name, and optional suffix. The prefix should be the Kind part of
+// TaskKind the envoy is being configured for.
+func PortLabel(prefix, service, suffix string) string {
+ if suffix == "" {
+ return fmt.Sprintf("%s-%s", prefix, service)
+ }
+ return fmt.Sprintf("%s-%s-%s", prefix, service, suffix)
+}
diff --git a/helper/envoy/envoy_test.go b/helper/envoy/envoy_test.go
new file mode 100644
index 00000000000..87a979c8586
--- /dev/null
+++ b/helper/envoy/envoy_test.go
@@ -0,0 +1,29 @@
+package envoy
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/hashicorp/nomad/nomad/structs"
+ "github.com/stretchr/testify/require"
+)
+
+func TestEnvoy_PortLabel(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ prefix string
+ service string
+ suffix string
+ exp string
+ }{
+ {prefix: structs.ConnectProxyPrefix, service: "foo", suffix: "", exp: "connect-proxy-foo"},
+ {prefix: structs.ConnectMeshPrefix, service: "bar", exp: "connect-mesh-bar"},
+ } {
+ test := fmt.Sprintf("%s_%s_%s", tc.prefix, tc.service, tc.suffix)
+ t.Run(test, func(t *testing.T) {
+ result := PortLabel(tc.prefix, tc.service, tc.suffix)
+ require.Equal(t, tc.exp, result)
+ })
+ }
+}
diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go
index 9add49fb83f..989039cde69 100644
--- a/nomad/job_endpoint_hook_connect.go
+++ b/nomad/job_endpoint_hook_connect.go
@@ -156,19 +156,20 @@ func getSidecarTaskForService(tg *structs.TaskGroup, svc string) *structs.Task {
return nil
}
-func isSidecarForService(t *structs.Task, svc string) bool {
- return t.Kind == structs.NewTaskKind(structs.ConnectProxyPrefix, svc)
+func isSidecarForService(t *structs.Task, service string) bool {
+ return t.Kind == structs.NewTaskKind(structs.ConnectProxyPrefix, service)
}
-func hasGatewayTaskForService(tg *structs.TaskGroup, svc string) bool {
+func hasGatewayTaskForService(tg *structs.TaskGroup, service string) bool {
for _, t := range tg.Tasks {
switch {
- case isIngressGatewayForService(t, svc):
+ case isIngressGatewayForService(t, service):
return true
- case isTerminatingGatewayForService(t, svc):
+ case isTerminatingGatewayForService(t, service):
+ return true
+ case isMeshGatewayForService(t, service):
return true
}
- // mesh later
}
return false
}
@@ -181,6 +182,10 @@ func isTerminatingGatewayForService(t *structs.Task, svc string) bool {
return t.Kind == structs.NewTaskKind(structs.ConnectTerminatingPrefix, svc)
}
+func isMeshGatewayForService(t *structs.Task, svc string) bool {
+ return t.Kind == structs.NewTaskKind(structs.ConnectMeshPrefix, svc)
+}
+
// getNamedTaskForNativeService retrieves the Task with the name specified in the
// group service definition. If the task name is empty and there is only one task
// in the group, infer the name from the only option.
@@ -218,8 +223,6 @@ func injectPort(group *structs.TaskGroup, label string) {
})
}
-// probably need to hack this up to look for checks on the service, and if they
-// qualify, configure a port for envoy to use to expose their paths.
func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
// Create an environment interpolator with what we have at submission time.
// This should only be used to interpolate connect service names which are
@@ -267,7 +270,7 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
// create a port for the sidecar task's proxy port
portLabel := service.Connect.SidecarService.Port
if portLabel == "" {
- portLabel = fmt.Sprintf("%s-%s", structs.ConnectProxyPrefix, service.Name)
+ portLabel = envoy.PortLabel(structs.ConnectProxyPrefix, service.Name, "")
}
injectPort(g, portLabel)
@@ -302,11 +305,25 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {
// reasonable to keep the magic alive here.
if service.Connect.IsTerminating() && service.PortLabel == "" {
// Inject a dynamic port for the terminating gateway.
- portLabel := fmt.Sprintf("%s-%s", structs.ConnectTerminatingPrefix, service.Name)
+ portLabel := envoy.PortLabel(structs.ConnectTerminatingPrefix, service.Name, "")
service.PortLabel = portLabel
injectPort(g, portLabel)
}
+ // A mesh Gateway will need 2 ports (lan and wan).
+ if service.Connect.IsMesh() {
+
+ // service port is used for mesh gateway wan address - it should
+ // come from a configured host_network to make sense
+ if service.PortLabel == "" {
+ return errors.New("service.port must be set for mesh gateway service")
+ }
+
+ // Inject a dynamic port for mesh gateway LAN address.
+ lanPortLabel := envoy.PortLabel(structs.ConnectMeshPrefix, service.Name, "lan")
+ injectPort(g, lanPortLabel)
+ }
+
// inject the gateway task only if it does not yet already exist
if !hasGatewayTaskForService(g, service.Name) {
prefix := service.Connect.Gateway.Prefix()
@@ -391,8 +408,20 @@ func gatewayProxyForBridge(gateway *structs.ConsulGateway) *structs.ConsulGatewa
Address: "0.0.0.0",
Port: -1, // filled in later with dynamic port
}}
+ case gateway.Mesh != nil:
+ proxy.EnvoyGatewayNoDefaultBind = true
+ proxy.EnvoyGatewayBindTaggedAddresses = false
+ proxy.EnvoyGatewayBindAddresses = map[string]*structs.ConsulGatewayBindAddress{
+ "wan": {
+ Address: "0.0.0.0",
+ Port: -1, // filled in later with configured port
+ },
+ "lan": {
+ Address: "0.0.0.0",
+ Port: -1, // filled in later with generated port
+ },
+ }
}
- // later: mesh
return proxy
}
diff --git a/nomad/job_endpoint_hook_connect_test.go b/nomad/job_endpoint_hook_connect_test.go
index d5df76a7278..3691eb58030 100644
--- a/nomad/job_endpoint_hook_connect_test.go
+++ b/nomad/job_endpoint_hook_connect_test.go
@@ -16,9 +16,9 @@ func TestJobEndpointConnect_isSidecarForService(t *testing.T) {
t.Parallel()
cases := []struct {
- t *structs.Task // task
- s string // service
- r bool // result
+ task *structs.Task
+ sidecar string
+ exp bool
}{
{
&structs.Task{},
@@ -49,7 +49,7 @@ func TestJobEndpointConnect_isSidecarForService(t *testing.T) {
}
for _, c := range cases {
- require.Equal(t, c.r, isSidecarForService(c.t, c.s))
+ require.Equal(t, c.exp, isSidecarForService(c.task, c.sidecar))
}
}
@@ -115,17 +115,16 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) {
func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) {
t.Parallel()
- // Test that the connect gateway task is inserted if a gateway service exists
- // and since this is a bridge network, will rewrite the default gateway proxy
+ // Test that the connect ingress gateway task is inserted if a gateway service
+ // exists and since this is a bridge network, will rewrite the default gateway proxy
// block with correct configuration.
job := mock.ConnectIngressGatewayJob("bridge", false)
-
job.Meta = map[string]string{
"gateway_name": "my-gateway",
}
-
job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}"
+ // setup expectations
expTG := job.TaskGroups[0].Copy()
expTG.Tasks = []*structs.Task{
// inject the gateway task
@@ -153,11 +152,9 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testin
// and since this is a bridge network, will rewrite the default gateway proxy
// block with correct configuration.
job := mock.ConnectIngressGatewayJob("bridge", false)
-
job.Meta = map[string]string{
"gateway_name": "my-gateway",
}
-
job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}"
job.TaskGroups[0].Services[0].Connect.SidecarTask = &structs.SidecarTask{
Driver: "raw_exec",
@@ -173,6 +170,7 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testin
KillSignal: "SIGHUP",
}
+ // setup expectations
expTG := job.TaskGroups[0].Copy()
expTG.Tasks = []*structs.Task{
// inject merged gateway task
@@ -217,6 +215,80 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testin
require.Exactly(t, expTG, job.TaskGroups[0])
}
+func TestJobEndpointConnect_groupConnectHook_TerminatingGateway(t *testing.T) {
+ t.Parallel()
+
+ // Tests that the connect terminating gateway task is inserted if a gateway
+ // service exists and since this is a bridge network, will rewrite the default
+ // gateway proxy block with correct configuration.
+ job := mock.ConnectTerminatingGatewayJob("bridge", false)
+ job.Meta = map[string]string{
+ "gateway_name": "my-gateway",
+ }
+ job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}"
+
+ // setup expectations
+ expTG := job.TaskGroups[0].Copy()
+ expTG.Tasks = []*structs.Task{
+ // inject the gateway task
+ newConnectGatewayTask(structs.ConnectTerminatingPrefix, "my-gateway", false),
+ }
+ expTG.Services[0].Name = "my-gateway"
+ expTG.Tasks[0].Canonicalize(job, expTG)
+ expTG.Networks[0].Canonicalize()
+
+ // rewrite the service gateway proxy configuration
+ expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway)
+
+ require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+ require.Exactly(t, expTG, job.TaskGroups[0])
+
+ // Test that the hook is idempotent
+ require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+ require.Exactly(t, expTG, job.TaskGroups[0])
+}
+
+func TestJobEndpointConnect_groupConnectHook_MeshGateway(t *testing.T) {
+ t.Parallel()
+
+ // Test that the connect mesh gateway task is inserted if a gateway service
+ // exists and since this is a bridge network, will rewrite the default gateway
+ // proxy block with correct configuration, injecting a dynamic port for use
+ // by the envoy lan listener.
+ job := mock.ConnectMeshGatewayJob("bridge", false)
+ job.Meta = map[string]string{
+ "gateway_name": "my-gateway",
+ }
+ job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}"
+
+ // setup expectations
+ expTG := job.TaskGroups[0].Copy()
+ expTG.Tasks = []*structs.Task{
+ // inject the gateway task
+ newConnectGatewayTask(structs.ConnectMeshPrefix, "my-gateway", false),
+ }
+ expTG.Services[0].Name = "my-gateway"
+ expTG.Services[0].PortLabel = "public_port"
+ expTG.Networks[0].DynamicPorts = []structs.Port{{
+ Label: "connect-mesh-my-gateway-lan",
+ Value: 0,
+ To: -1,
+ HostNetwork: "default",
+ }}
+ expTG.Tasks[0].Canonicalize(job, expTG)
+ expTG.Networks[0].Canonicalize()
+
+ // rewrite the service gateway proxy configuration
+ expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway)
+
+ require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+ require.Exactly(t, expTG, job.TaskGroups[0])
+
+ // Test that the hook is idempotent
+ require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+ require.Exactly(t, expTG, job.TaskGroups[0])
+}
+
// TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar
// proxy task is being created for a group service with an interpolated name,
// the service name is interpolated *before the task is created.
@@ -395,6 +467,8 @@ func TestJobEndpointConnect_groupConnectGatewayValidate(t *testing.T) {
}
func TestJobEndpointConnect_newConnectGatewayTask_host(t *testing.T) {
+ t.Parallel()
+
t.Run("ingress", func(t *testing.T) {
task := newConnectGatewayTask(structs.ConnectIngressPrefix, "foo", true)
require.Equal(t, "connect-ingress-foo", task.Name)
@@ -415,11 +489,15 @@ func TestJobEndpointConnect_newConnectGatewayTask_host(t *testing.T) {
}
func TestJobEndpointConnect_newConnectGatewayTask_bridge(t *testing.T) {
+ t.Parallel()
+
task := newConnectGatewayTask(structs.ConnectIngressPrefix, "service1", false)
require.NotContains(t, task.Config, "network_mode")
}
func TestJobEndpointConnect_hasGatewayTaskForService(t *testing.T) {
+ t.Parallel()
+
t.Run("no gateway task", func(t *testing.T) {
result := hasGatewayTaskForService(&structs.TaskGroup{
Name: "group",
@@ -452,9 +530,22 @@ func TestJobEndpointConnect_hasGatewayTaskForService(t *testing.T) {
}, "my-service")
require.True(t, result)
})
+
+ t.Run("has mesh task", func(t *testing.T) {
+ result := hasGatewayTaskForService(&structs.TaskGroup{
+ Name: "group",
+ Tasks: []*structs.Task{{
+ Name: "mesh-gateway-my-service",
+ Kind: structs.NewTaskKind(structs.ConnectMeshPrefix, "my-service"),
+ }},
+ }, "my-service")
+ require.True(t, result)
+ })
}
func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) {
+ t.Parallel()
+
t.Run("nil", func(t *testing.T) {
result := gatewayProxyIsDefault(nil)
require.True(t, result)
@@ -496,6 +587,8 @@ func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) {
}
func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
+ t.Parallel()
+
t.Run("nil", func(t *testing.T) {
result := gatewayBindAddressesIngress(nil)
@@ -561,6 +654,8 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
}
func TestJobEndpointConnect_gatewayProxyForBridge(t *testing.T) {
+ t.Parallel()
+
t.Run("nil", func(t *testing.T) {
result := gatewayProxyForBridge(nil)
require.Nil(t, result)
@@ -636,6 +731,7 @@ func TestJobEndpointConnect_gatewayProxyForBridge(t *testing.T) {
},
})
require.Equal(t, &structs.ConsulGatewayProxy{
+ ConnectTimeout: nil,
Config: map[string]interface{}{"foo": 1},
EnvoyGatewayNoDefaultBind: false,
EnvoyGatewayBindTaggedAddresses: true,
@@ -674,6 +770,69 @@ func TestJobEndpointConnect_gatewayProxyForBridge(t *testing.T) {
})
t.Run("terminating leave as-is", func(t *testing.T) {
- //
+ result := gatewayProxyForBridge(&structs.ConsulGateway{
+ Proxy: &structs.ConsulGatewayProxy{
+ Config: map[string]interface{}{"foo": 1},
+ EnvoyGatewayBindTaggedAddresses: true,
+ },
+ Terminating: &structs.ConsulTerminatingConfigEntry{
+ Services: []*structs.ConsulLinkedService{{
+ Name: "service1",
+ }},
+ },
+ })
+ require.Equal(t, &structs.ConsulGatewayProxy{
+ ConnectTimeout: nil,
+ Config: map[string]interface{}{"foo": 1},
+ EnvoyGatewayNoDefaultBind: false,
+ EnvoyGatewayBindTaggedAddresses: true,
+ EnvoyGatewayBindAddresses: nil,
+ }, result)
+ })
+
+ t.Run("mesh set defaults", func(t *testing.T) {
+ result := gatewayProxyForBridge(&structs.ConsulGateway{
+ Proxy: &structs.ConsulGatewayProxy{
+ ConnectTimeout: helper.TimeToPtr(2 * time.Second),
+ },
+ Mesh: &structs.ConsulMeshConfigEntry{
+ // nothing
+ },
+ })
+ require.Equal(t, &structs.ConsulGatewayProxy{
+ ConnectTimeout: helper.TimeToPtr(2 * time.Second),
+ EnvoyGatewayNoDefaultBind: true,
+ EnvoyGatewayBindTaggedAddresses: false,
+ EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
+ "lan": {
+ Address: "0.0.0.0",
+ Port: -1,
+ },
+ "wan": {
+ Address: "0.0.0.0",
+ Port: -1,
+ },
+ },
+ }, result)
})
+
+ t.Run("mesh leave as-is", func(t *testing.T) {
+ result := gatewayProxyForBridge(&structs.ConsulGateway{
+ Proxy: &structs.ConsulGatewayProxy{
+ Config: map[string]interface{}{"foo": 1},
+ EnvoyGatewayBindTaggedAddresses: true,
+ },
+ Mesh: &structs.ConsulMeshConfigEntry{
+ // nothing
+ },
+ })
+ require.Equal(t, &structs.ConsulGatewayProxy{
+ ConnectTimeout: nil,
+ Config: map[string]interface{}{"foo": 1},
+ EnvoyGatewayNoDefaultBind: false,
+ EnvoyGatewayBindTaggedAddresses: true,
+ EnvoyGatewayBindAddresses: nil,
+ }, result)
+ })
+
}
diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go
index 6a54a60d085..ea9cb512900 100644
--- a/nomad/mock/mock.go
+++ b/nomad/mock/mock.go
@@ -899,6 +899,9 @@ func ConnectIngressGatewayJob(mode string, inject bool) *structs.Job {
},
},
}}
+
+ tg.Tasks = nil
+
// some tests need to assume the gateway proxy task has already been injected
if inject {
tg.Tasks = []*structs.Task{{
@@ -912,9 +915,102 @@ func ConnectIngressGatewayJob(mode string, inject bool) *structs.Job {
MaxFileSizeMB: 2,
},
}}
- } else {
- // otherwise there are no tasks in the group yet
- tg.Tasks = nil
+ }
+ return job
+}
+
+// ConnectTerminatingGatewayJob creates a structs.Job that contains the definition
+// of a Consul Terminating Gateway service. The mode is the name of the network mode
+// assumed by the task group. If inject is true, a corresponding task is set on the
+// group's Tasks (i.e. what the job would look like after mutation).
+func ConnectTerminatingGatewayJob(mode string, inject bool) *structs.Job {
+ job := Job()
+ tg := job.TaskGroups[0]
+ tg.Networks = []*structs.NetworkResource{{
+ Mode: mode,
+ }}
+ tg.Services = []*structs.Service{{
+ Name: "my-terminating-service",
+ PortLabel: "9999",
+ Connect: &structs.ConsulConnect{
+ Gateway: &structs.ConsulGateway{
+ Proxy: &structs.ConsulGatewayProxy{
+ ConnectTimeout: helper.TimeToPtr(3 * time.Second),
+ EnvoyGatewayBindAddresses: make(map[string]*structs.ConsulGatewayBindAddress),
+ },
+ Terminating: &structs.ConsulTerminatingConfigEntry{
+ Services: []*structs.ConsulLinkedService{{
+ Name: "service1",
+ CAFile: "/ssl/ca_file",
+ CertFile: "/ssl/cert_file",
+ KeyFile: "/ssl/key_file",
+ SNI: "sni-name",
+ }},
+ },
+ },
+ },
+ }}
+
+ tg.Tasks = nil
+
+ // some tests need to assume the gateway proxy task has already been injected
+ if inject {
+ tg.Tasks = []*structs.Task{{
+ Name: fmt.Sprintf("%s-%s", structs.ConnectTerminatingPrefix, "my-terminating-service"),
+ Kind: structs.NewTaskKind(structs.ConnectTerminatingPrefix, "my-terminating-service"),
+ Driver: "docker",
+ Config: make(map[string]interface{}),
+ ShutdownDelay: 5 * time.Second,
+ LogConfig: &structs.LogConfig{
+ MaxFiles: 2,
+ MaxFileSizeMB: 2,
+ },
+ }}
+ }
+ return job
+}
+
+// ConnectMeshGatewayJob creates a structs.Job that contains the definition of a
+// Consul Mesh Gateway service. The mode is the name of the network mode assumed
+// by the task group. If inject is true, a corresponding task is set on the group's
+// Tasks (i.e. what the job would look like after job mutation).
+func ConnectMeshGatewayJob(mode string, inject bool) *structs.Job {
+ job := Job()
+ tg := job.TaskGroups[0]
+ tg.Networks = []*structs.NetworkResource{{
+ Mode: mode,
+ }}
+ tg.Services = []*structs.Service{{
+ Name: "my-mesh-service",
+ PortLabel: "public_port",
+ Connect: &structs.ConsulConnect{
+ Gateway: &structs.ConsulGateway{
+ Proxy: &structs.ConsulGatewayProxy{
+ ConnectTimeout: helper.TimeToPtr(3 * time.Second),
+ EnvoyGatewayBindAddresses: make(map[string]*structs.ConsulGatewayBindAddress),
+ },
+ Mesh: &structs.ConsulMeshConfigEntry{
+ // nothing to configure
+ },
+ },
+ },
+ }}
+
+ tg.Tasks = nil
+
+ // some tests need to assume the gateway task has already been injected
+ if inject {
+ tg.Tasks = []*structs.Task{{
+ Name: fmt.Sprintf("%s-%s", structs.ConnectMeshPrefix, "my-mesh-service"),
+ Kind: structs.NewTaskKind(structs.ConnectMeshPrefix, "my-mesh-service"),
+ Driver: "docker",
+ Config: make(map[string]interface{}),
+ ShutdownDelay: 5 * time.Second,
+ LogConfig: &structs.LogConfig{
+ MaxFiles: 2,
+ MaxFileSizeMB: 2,
+ },
+ }}
}
return job
}
diff --git a/nomad/structs/connect_test.go b/nomad/structs/connect_test.go
new file mode 100644
index 00000000000..385716fe8ba
--- /dev/null
+++ b/nomad/structs/connect_test.go
@@ -0,0 +1,21 @@
+package structs
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestTaskKind_IsAnyConnectGateway(t *testing.T) {
+ t.Run("gateways", func(t *testing.T) {
+ require.True(t, NewTaskKind(ConnectIngressPrefix, "foo").IsAnyConnectGateway())
+ require.True(t, NewTaskKind(ConnectTerminatingPrefix, "foo").IsAnyConnectGateway())
+ require.True(t, NewTaskKind(ConnectMeshPrefix, "foo").IsAnyConnectGateway())
+ })
+
+ t.Run("not gateways", func(t *testing.T) {
+ require.False(t, NewTaskKind(ConnectProxyPrefix, "foo").IsAnyConnectGateway())
+ require.False(t, NewTaskKind(ConnectNativePrefix, "foo").IsAnyConnectGateway())
+ require.False(t, NewTaskKind("", "foo").IsAnyConnectGateway())
+ })
+}
diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go
index 544af3e630d..24d1f242e14 100644
--- a/nomad/structs/diff.go
+++ b/nomad/structs/diff.go
@@ -849,6 +849,32 @@ func connectGatewayDiff(prev, next *ConsulGateway, contextual bool) *ObjectDiff
diff.Objects = append(diff.Objects, gatewayTerminatingDiff)
}
+ // Diff the mesh gateway fields.
+ gatewayMeshDiff := connectGatewayMeshDiff(prev.Mesh, next.Mesh, contextual)
+ if gatewayMeshDiff != nil {
+ diff.Objects = append(diff.Objects, gatewayMeshDiff)
+ }
+
+ return diff
+}
+
+func connectGatewayMeshDiff(prev, next *ConsulMeshConfigEntry, contextual bool) *ObjectDiff {
+ diff := &ObjectDiff{Type: DiffTypeNone, Name: "Mesh"}
+
+ if reflect.DeepEqual(prev, next) {
+ return nil
+ } else if prev == nil {
+ // no fields to further diff
+ diff.Type = DiffTypeAdded
+ } else if next == nil {
+ // no fields to further diff
+ diff.Type = DiffTypeDeleted
+ } else {
+ diff.Type = DiffTypeEdited
+ }
+
+ // Currently no fields in mesh gateways.
+
return diff
}
@@ -1326,18 +1352,15 @@ func consulProxyDiff(old, new *ConsulProxy, contextual bool) *ObjectDiff {
newPrimitiveFlat = flatmap.Flatten(new, nil, true)
}
- // Diff the primitive fields.
+ // diff the primitive fields
diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
- consulUpstreamsDiff := primitiveObjectSetDiff(
- interfaceSlice(old.Upstreams),
- interfaceSlice(new.Upstreams),
- nil, "ConsulUpstreams", contextual)
- if consulUpstreamsDiff != nil {
- diff.Objects = append(diff.Objects, consulUpstreamsDiff...)
+ // diff the consul upstream slices
+ if upDiffs := consulProxyUpstreamsDiff(old.Upstreams, new.Upstreams, contextual); upDiffs != nil {
+ diff.Objects = append(diff.Objects, upDiffs...)
}
- // Config diff
+ // diff the config blob
if cDiff := configDiff(old.Config, new.Config, contextual); cDiff != nil {
diff.Objects = append(diff.Objects, cDiff)
}
@@ -1345,6 +1368,74 @@ func consulProxyDiff(old, new *ConsulProxy, contextual bool) *ObjectDiff {
return diff
}
+// consulProxyUpstreamsDiff diffs a set of connect upstreams. If contextual diff is
+// enabled, unchanged fields within objects nested in the tasks will be returned.
+func consulProxyUpstreamsDiff(old, new []ConsulUpstream, contextual bool) []*ObjectDiff {
+ oldMap := make(map[string]ConsulUpstream, len(old))
+ newMap := make(map[string]ConsulUpstream, len(new))
+
+ idx := func(up ConsulUpstream) string {
+ return fmt.Sprintf("%s/%s", up.Datacenter, up.DestinationName)
+ }
+
+ for _, o := range old {
+ oldMap[idx(o)] = o
+ }
+ for _, n := range new {
+ newMap[idx(n)] = n
+ }
+
+ var diffs []*ObjectDiff
+ for index, oldUpstream := range oldMap {
+ // Diff the same, deleted, and edited
+ if diff := consulProxyUpstreamDiff(oldUpstream, newMap[index], contextual); diff != nil {
+ diffs = append(diffs, diff)
+ }
+ }
+
+ for index, newUpstream := range newMap {
+ // diff the added
+ if oldUpstream, exists := oldMap[index]; !exists {
+ if diff := consulProxyUpstreamDiff(oldUpstream, newUpstream, contextual); diff != nil {
+ diffs = append(diffs, diff)
+ }
+ }
+ }
+ sort.Sort(ObjectDiffs(diffs))
+ return diffs
+}
+
+func consulProxyUpstreamDiff(prev, next ConsulUpstream, contextual bool) *ObjectDiff {
+ diff := &ObjectDiff{Type: DiffTypeNone, Name: "ConsulUpstreams"}
+ var oldPrimFlat, newPrimFlat map[string]string
+
+ if reflect.DeepEqual(prev, next) {
+ return nil
+ } else if prev.Equals(new(ConsulUpstream)) {
+ prev = ConsulUpstream{}
+ diff.Type = DiffTypeAdded
+ newPrimFlat = flatmap.Flatten(next, nil, true)
+ } else if next.Equals(new(ConsulUpstream)) {
+ next = ConsulUpstream{}
+ diff.Type = DiffTypeDeleted
+ oldPrimFlat = flatmap.Flatten(prev, nil, true)
+ } else {
+ diff.Type = DiffTypeEdited
+ oldPrimFlat = flatmap.Flatten(prev, nil, true)
+ newPrimFlat = flatmap.Flatten(next, nil, true)
+ }
+
+ // diff the primitive fields
+ diff.Fields = fieldDiffs(oldPrimFlat, newPrimFlat, contextual)
+
+ // diff the mesh gateway primitive object
+ if mDiff := primitiveObjectDiff(prev.MeshGateway, next.MeshGateway, nil, "MeshGateway", contextual); mDiff != nil {
+ diff.Objects = append(diff.Objects, mDiff)
+ }
+
+ return diff
+}
+
// serviceCheckDiffs diffs a set of service checks. If contextual diff is
// enabled, unchanged fields within objects nested in the tasks will be
// returned.
diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go
index 7e21be5544b..ee6a7d06e17 100644
--- a/nomad/structs/diff_test.go
+++ b/nomad/structs/diff_test.go
@@ -2742,6 +2742,9 @@ func TestTaskGroupDiff(t *testing.T) {
SNI: "linked1.consul",
}},
},
+ Mesh: &ConsulMeshConfigEntry{
+ // nothing
+ },
},
},
},
@@ -2785,6 +2788,9 @@ func TestTaskGroupDiff(t *testing.T) {
LocalBindPort: 8000,
Datacenter: "dc2",
LocalBindAddress: "127.0.0.2",
+ MeshGateway: &ConsulMeshGateway{
+ Mode: "remote",
+ },
},
},
Config: map[string]interface{}{
@@ -2830,6 +2836,9 @@ func TestTaskGroupDiff(t *testing.T) {
SNI: "linked2.consul",
}},
},
+ Mesh: &ConsulMeshConfigEntry{
+ // nothing
+ },
},
},
},
@@ -3104,6 +3113,20 @@ func TestTaskGroupDiff(t *testing.T) {
New: "8000",
},
},
+ Objects: []*ObjectDiff{
+ {
+ Type: DiffTypeAdded,
+ Name: "MeshGateway",
+ Fields: []*FieldDiff{
+ {
+ Type: DiffTypeAdded,
+ Name: "Mode",
+ Old: "",
+ New: "remote",
+ },
+ },
+ },
+ },
},
{
Type: DiffTypeAdded,
diff --git a/nomad/structs/services.go b/nomad/structs/services.go
index 51202a89fde..8b1afa61e30 100644
--- a/nomad/structs/services.go
+++ b/nomad/structs/services.go
@@ -829,7 +829,9 @@ func (c *ConsulConnect) IsTerminating() bool {
return c.IsGateway() && c.Gateway.Terminating != nil
}
-// also mesh
+func (c *ConsulConnect) IsMesh() bool {
+ return c.IsGateway() && c.Gateway.Mesh != nil
+}
// Validate that the Connect block represents exactly one of:
// - Connect non-native service sidecar proxy
@@ -1221,6 +1223,56 @@ func (p *ConsulProxy) Equals(o *ConsulProxy) bool {
return true
}
+// ConsulMeshGateway is used to configure mesh gateway usage when connecting to
+// a connect upstream in another datacenter.
+type ConsulMeshGateway struct {
+ // Mode configures how an upstream should be accessed with regard to using
+ // mesh gateways.
+ //
+ // local - the connect proxy makes outbound connections through mesh gateway
+ // originating in the same datacenter.
+ //
+ // remote - the connect proxy makes outbound connections to a mesh gateway
+ // in the destination datacenter.
+ //
+ // none (default) - no mesh gateway is used, the proxy makes outbound connections
+ // directly to destination services.
+ //
+ // https://www.consul.io/docs/connect/gateways/mesh-gateway#modes-of-operation
+ Mode string
+}
+
+func (c *ConsulMeshGateway) Copy() *ConsulMeshGateway {
+ if c == nil {
+ return nil
+ }
+
+ return &ConsulMeshGateway{
+ Mode: c.Mode,
+ }
+}
+
+func (c *ConsulMeshGateway) Equals(o *ConsulMeshGateway) bool {
+ if c == nil || o == nil {
+ return c == o
+ }
+
+ return c.Mode == o.Mode
+}
+
+func (c *ConsulMeshGateway) Validate() error {
+ if c == nil {
+ return nil
+ }
+
+ switch c.Mode {
+ case "local", "remote", "none":
+ return nil
+ default:
+ return fmt.Errorf("Connect mesh_gateway mode %q not supported", c.Mode)
+ }
+}
+
// ConsulUpstream represents a Consul Connect upstream jobspec stanza.
type ConsulUpstream struct {
// DestinationName is the name of the upstream service.
@@ -1236,6 +1288,10 @@ type ConsulUpstream struct {
// LocalBindAddress is the address the proxy will receive connections for the
// upstream on.
LocalBindAddress string
+
+ // MeshGateway is the optional configuration of the mesh gateway for this
+ // upstream to use.
+ MeshGateway *ConsulMeshGateway
}
func upstreamsEquals(a, b []ConsulUpstream) bool {
@@ -1266,6 +1322,7 @@ func (u *ConsulUpstream) Copy() *ConsulUpstream {
LocalBindPort: u.LocalBindPort,
Datacenter: u.Datacenter,
LocalBindAddress: u.LocalBindAddress,
+ MeshGateway: u.MeshGateway.Copy(),
}
}
@@ -1275,10 +1332,23 @@ func (u *ConsulUpstream) Equals(o *ConsulUpstream) bool {
return u == o
}
- return (*u) == (*o)
+ switch {
+ case u.DestinationName != o.DestinationName:
+ return false
+ case u.LocalBindPort != o.LocalBindPort:
+ return false
+ case u.Datacenter != o.Datacenter:
+ return false
+ case u.LocalBindAddress != o.LocalBindAddress:
+ return false
+ case !u.MeshGateway.Equals(o.MeshGateway):
+ return false
+ }
+
+ return true
}
-// ExposeConfig represents a Consul Connect expose jobspec stanza.
+// ConsulExposeConfig represents a Consul Connect expose jobspec stanza.
type ConsulExposeConfig struct {
// Use json tag to match with field name in api/
Paths []ConsulExposePath `json:"Path"`
@@ -1341,18 +1411,19 @@ type ConsulGateway struct {
// Terminating represents the Consul Configuration Entry for a Terminating Gateway.
Terminating *ConsulTerminatingConfigEntry
- // Mesh is not yet supported.
- // Mesh *ConsulMeshConfigEntry
+ // Mesh indicates the Consul service should be a Mesh Gateway.
+ Mesh *ConsulMeshConfigEntry
}
func (g *ConsulGateway) Prefix() string {
switch {
+ case g.Mesh != nil:
+ return ConnectMeshPrefix
case g.Ingress != nil:
return ConnectIngressPrefix
default:
return ConnectTerminatingPrefix
}
- // also mesh
}
func (g *ConsulGateway) Copy() *ConsulGateway {
@@ -1364,6 +1435,7 @@ func (g *ConsulGateway) Copy() *ConsulGateway {
Proxy: g.Proxy.Copy(),
Ingress: g.Ingress.Copy(),
Terminating: g.Terminating.Copy(),
+ Mesh: g.Mesh.Copy(),
}
}
@@ -1384,6 +1456,10 @@ func (g *ConsulGateway) Equals(o *ConsulGateway) bool {
return false
}
+ if !g.Mesh.Equals(o.Mesh) {
+ return false
+ }
+
return true
}
@@ -1404,7 +1480,11 @@ func (g *ConsulGateway) Validate() error {
return err
}
- // Exactly 1 of ingress/terminating/mesh(soon) must be set.
+ if err := g.Mesh.Validate(); err != nil {
+ return err
+ }
+
+ // Exactly 1 of ingress/terminating/mesh must be set.
count := 0
if g.Ingress != nil {
count++
@@ -1412,8 +1492,11 @@ func (g *ConsulGateway) Validate() error {
if g.Terminating != nil {
count++
}
+ if g.Mesh != nil {
+ count++
+ }
if count != 1 {
- return fmt.Errorf("One Consul Gateway Configuration Entry must be set")
+ return fmt.Errorf("One Consul Gateway Configuration must be set")
}
return nil
}
@@ -1612,11 +1695,7 @@ func (c *ConsulGatewayTLSConfig) Equals(o *ConsulGatewayTLSConfig) bool {
// ConsulIngressService is used to configure a service fronted by the ingress gateway.
type ConsulIngressService struct {
- // Namespace is not yet supported.
- // Namespace string
-
- Name string
-
+ Name string
Hosts []string
}
@@ -1776,9 +1855,6 @@ COMPARE: // order does not matter
//
// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields
type ConsulIngressConfigEntry struct {
- // Namespace is not yet supported.
- // Namespace string
-
TLS *ConsulGatewayTLSConfig
Listeners []*ConsulIngressListener
}
@@ -1939,9 +2015,6 @@ COMPARE: // order does not matter
}
type ConsulTerminatingConfigEntry struct {
- // Namespace is not yet supported.
- // Namespace string
-
Services []*ConsulLinkedService
}
@@ -1988,3 +2061,30 @@ func (e *ConsulTerminatingConfigEntry) Validate() error {
return nil
}
+
+// ConsulMeshConfigEntry is a stub used to represent that the gateway service
+// type should be for a Mesh Gateway. Unlike Ingress and Terminating, there is no
+// dedicated Consul Config Entry type for "mesh-gateway", for now. We still
+// create a type for future proofing, and to keep underlying job-spec marshaling
+// consistent with the other types.
+type ConsulMeshConfigEntry struct {
+ // nothing in here
+}
+
+func (e *ConsulMeshConfigEntry) Copy() *ConsulMeshConfigEntry {
+ if e == nil {
+ return nil
+ }
+ return new(ConsulMeshConfigEntry)
+}
+
+func (e *ConsulMeshConfigEntry) Equals(o *ConsulMeshConfigEntry) bool {
+ if e == nil || o == nil {
+ return e == o
+ }
+ return true
+}
+
+func (e *ConsulMeshConfigEntry) Validate() error {
+ return nil
+}
diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go
index 6535f8c2e2f..49e31b502dc 100644
--- a/nomad/structs/services_test.go
+++ b/nomad/structs/services_test.go
@@ -522,6 +522,12 @@ func TestConsulUpstream_upstreamEquals(t *testing.T) {
require.False(t, upstreamsEquals(a, b))
})
+ t.Run("different mesh_gateway", func(t *testing.T) {
+ a := []ConsulUpstream{{DestinationName: "foo", MeshGateway: &ConsulMeshGateway{Mode: "local"}}}
+ b := []ConsulUpstream{{DestinationName: "foo", MeshGateway: &ConsulMeshGateway{Mode: "remote"}}}
+ require.False(t, upstreamsEquals(a, b))
+ })
+
t.Run("identical", func(t *testing.T) {
a := []ConsulUpstream{up("foo", 8000), up("bar", 9000)}
b := []ConsulUpstream{up("foo", 8000), up("bar", 9000)}
@@ -682,6 +688,15 @@ var (
}},
},
}
+
+ consulMeshGateway1 = &ConsulGateway{
+ Proxy: &ConsulGatewayProxy{
+ ConnectTimeout: helper.TimeToPtr(1 * time.Second),
+ },
+ Mesh: &ConsulMeshConfigEntry{
+ // nothing
+ },
+ }
)
func TestConsulGateway_Prefix(t *testing.T) {
@@ -695,7 +710,10 @@ func TestConsulGateway_Prefix(t *testing.T) {
require.Equal(t, ConnectTerminatingPrefix, result)
})
- // also mesh
+ t.Run("mesh", func(t *testing.T) {
+ result := (&ConsulGateway{Mesh: new(ConsulMeshConfigEntry)}).Prefix()
+ require.Equal(t, ConnectMeshPrefix, result)
+ })
}
func TestConsulGateway_Copy(t *testing.T) {
@@ -720,6 +738,29 @@ func TestConsulGateway_Copy(t *testing.T) {
require.True(t, result.Equals(consulTerminatingGateway1))
require.True(t, consulTerminatingGateway1.Equals(result))
})
+
+ t.Run("as mesh", func(t *testing.T) {
+ result := consulMeshGateway1.Copy()
+ require.Equal(t, consulMeshGateway1, result)
+ require.True(t, result.Equals(consulMeshGateway1))
+ require.True(t, consulMeshGateway1.Equals(result))
+ })
+}
+
+func TestConsulGateway_Equals_mesh(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ a := (*ConsulGateway)(nil)
+ b := (*ConsulGateway)(nil)
+ require.True(t, a.Equals(b))
+ require.False(t, a.Equals(consulMeshGateway1))
+ require.False(t, consulMeshGateway1.Equals(a))
+ })
+
+ t.Run("reflexive", func(t *testing.T) {
+ require.True(t, consulMeshGateway1.Equals(consulMeshGateway1))
+ })
}
func TestConsulGateway_Equals_ingress(t *testing.T) {
@@ -962,8 +1003,9 @@ func TestConsulGateway_Validate(t *testing.T) {
err := (&ConsulGateway{
Ingress: nil,
Terminating: nil,
+ Mesh: nil,
}).Validate()
- require.EqualError(t, err, "One Consul Gateway Configuration Entry must be set")
+ require.EqualError(t, err, "One Consul Gateway Configuration must be set")
})
t.Run("multiple config entries set", func(t *testing.T) {
@@ -983,7 +1025,14 @@ func TestConsulGateway_Validate(t *testing.T) {
}},
},
}).Validate()
- require.EqualError(t, err, "One Consul Gateway Configuration Entry must be set")
+ require.EqualError(t, err, "One Consul Gateway Configuration must be set")
+ })
+
+ t.Run("ok mesh", func(t *testing.T) {
+ err := (&ConsulGateway{
+ Mesh: new(ConsulMeshConfigEntry),
+ }).Validate()
+ require.NoError(t, err)
})
}
@@ -1227,7 +1276,7 @@ func TestConsulLinkedService_Validate(t *testing.T) {
require.EqualError(t, err, "Consul Linked Service requires Name")
})
- t.Run("missing cafile", func(t *testing.T) {
+ t.Run("missing ca_file", func(t *testing.T) {
err := (&ConsulLinkedService{
Name: "linked-service1",
CertFile: "cert_file.pem",
@@ -1245,7 +1294,7 @@ func TestConsulLinkedService_Validate(t *testing.T) {
require.EqualError(t, err, "Consul Linked Service TLS Cert and Key must both be set")
})
- t.Run("sni without cafile", func(t *testing.T) {
+ t.Run("sni without ca_file", func(t *testing.T) {
err := (&ConsulLinkedService{
Name: "linked-service1",
SNI: "service.consul",
@@ -1339,7 +1388,7 @@ func TestConsulLinkedService_linkedServicesEqual(t *testing.T) {
require.True(t, linkedServicesEqual(services, reversed))
different := []*ConsulLinkedService{
- services[0], &ConsulLinkedService{
+ services[0], {
Name: "service2",
CAFile: "ca.pem",
SNI: "service2.consul",
@@ -1382,3 +1431,44 @@ func TestConsulTerminatingConfigEntry_Validate(t *testing.T) {
require.NoError(t, err)
})
}
+
+func TestConsulMeshGateway_Copy(t *testing.T) {
+ t.Parallel()
+
+ require.Nil(t, (*ConsulMeshGateway)(nil))
+ require.Equal(t, &ConsulMeshGateway{
+ Mode: "remote",
+ }, &ConsulMeshGateway{
+ Mode: "remote",
+ })
+}
+
+func TestConsulMeshGateway_Equals(t *testing.T) {
+ t.Parallel()
+
+ c := &ConsulMeshGateway{Mode: "local"}
+ require.False(t, c.Equals(nil))
+ require.True(t, c.Equals(c))
+
+ o := &ConsulMeshGateway{Mode: "remote"}
+ require.False(t, c.Equals(o))
+}
+
+func TestConsulMeshGateway_Validate(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nil", func(t *testing.T) {
+ err := (*ConsulMeshGateway)(nil).Validate()
+ require.NoError(t, err)
+ })
+
+ t.Run("mode invalid", func(t *testing.T) {
+ err := (&ConsulMeshGateway{Mode: "banana"}).Validate()
+ require.EqualError(t, err, `Connect mesh_gateway mode "banana" not supported`)
+ })
+
+ t.Run("ok", func(t *testing.T) {
+ err := (&ConsulMeshGateway{Mode: "local"}).Validate()
+ require.NoError(t, err)
+ })
+}
diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go
index bb61726ac57..e7c0f49aa78 100644
--- a/nomad/structs/structs.go
+++ b/nomad/structs/structs.go
@@ -7209,20 +7209,31 @@ func (k TaskKind) IsConnectNative() bool {
return k.hasPrefix(ConnectNativePrefix)
}
+// IsConnectIngress returns true if the TaskKind is connect-ingress.
func (k TaskKind) IsConnectIngress() bool {
return k.hasPrefix(ConnectIngressPrefix)
}
+// IsConnectTerminating returns true if the TaskKind is connect-terminating.
func (k TaskKind) IsConnectTerminating() bool {
return k.hasPrefix(ConnectTerminatingPrefix)
}
+// IsConnectMesh returns true if the TaskKind is connect-mesh.
+func (k TaskKind) IsConnectMesh() bool {
+ return k.hasPrefix(ConnectMeshPrefix)
+}
+
+// IsAnyConnectGateway returns true if the TaskKind represents any one of the
+// supported connect gateway types.
func (k TaskKind) IsAnyConnectGateway() bool {
switch {
case k.IsConnectIngress():
return true
case k.IsConnectTerminating():
return true
+ case k.IsConnectMesh():
+ return true
default:
return false
}
@@ -7243,14 +7254,11 @@ const (
// ConnectTerminatingPrefix is the prefix used for fields referencing a Consul
// Connect Terminating Gateway Proxy.
- //
ConnectTerminatingPrefix = "connect-terminating"
// ConnectMeshPrefix is the prefix used for fields referencing a Consul Connect
// Mesh Gateway Proxy.
- //
- // Not yet supported.
- // ConnectMeshPrefix = "connect-mesh"
+ ConnectMeshPrefix = "connect-mesh"
)
// ValidateConnectProxyService checks that the service that is being
diff --git a/vendor/github.com/hashicorp/nomad/api/services.go b/vendor/github.com/hashicorp/nomad/api/services.go
index 880ba7f7973..34440124c7a 100644
--- a/vendor/github.com/hashicorp/nomad/api/services.go
+++ b/vendor/github.com/hashicorp/nomad/api/services.go
@@ -281,17 +281,77 @@ func (cp *ConsulProxy) Canonicalize() {
cp.Upstreams = nil
}
+ for _, upstream := range cp.Upstreams {
+ upstream.Canonicalize()
+ }
+
if len(cp.Config) == 0 {
cp.Config = nil
}
}
+// ConsulMeshGateway is used to configure mesh gateway usage when connecting to
+// a connect upstream in another datacenter.
+type ConsulMeshGateway struct {
+ // Mode configures how an upstream should be accessed with regard to using
+ // mesh gateways.
+ //
+ // local - the connect proxy makes outbound connections through mesh gateway
+ // originating in the same datacenter.
+ //
+ // remote - the connect proxy makes outbound connections to a mesh gateway
+ // in the destination datacenter.
+ //
+ // none (default) - no mesh gateway is used, the proxy makes outbound connections
+ // directly to destination services.
+ //
+ // https://www.consul.io/docs/connect/gateways/mesh-gateway#modes-of-operation
+ Mode string `mapstructure:"mode" hcl:"mode,optional"`
+}
+
+func (c *ConsulMeshGateway) Canonicalize() {
+ // Mode may be empty string, indicating behavior will defer to Consul
+ // service-defaults config entry.
+ return
+}
+
+func (c *ConsulMeshGateway) Copy() *ConsulMeshGateway {
+ if c == nil {
+ return nil
+ }
+
+ return &ConsulMeshGateway{
+ Mode: c.Mode,
+ }
+}
+
// ConsulUpstream represents a Consul Connect upstream jobspec stanza.
type ConsulUpstream struct {
- DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"`
- LocalBindPort int `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"`
- Datacenter string `mapstructure:"datacenter" hcl:"datacenter,optional"`
- LocalBindAddress string `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"`
+ DestinationName string `mapstructure:"destination_name" hcl:"destination_name,optional"`
+ LocalBindPort int `mapstructure:"local_bind_port" hcl:"local_bind_port,optional"`
+ Datacenter string `mapstructure:"datacenter" hcl:"datacenter,optional"`
+ LocalBindAddress string `mapstructure:"local_bind_address" hcl:"local_bind_address,optional"`
+ MeshGateway *ConsulMeshGateway `mapstructure:"mesh_gateway" hcl:"mesh_gateway,block"`
+}
+
+func (cu *ConsulUpstream) Copy() *ConsulUpstream {
+ if cu == nil {
+ return nil
+ }
+ return &ConsulUpstream{
+ DestinationName: cu.DestinationName,
+ LocalBindPort: cu.LocalBindPort,
+ Datacenter: cu.Datacenter,
+ LocalBindAddress: cu.LocalBindAddress,
+ MeshGateway: cu.MeshGateway.Copy(),
+ }
+}
+
+func (cu *ConsulUpstream) Canonicalize() {
+ if cu == nil {
+ return
+ }
+ cu.MeshGateway.Canonicalize()
}
type ConsulExposeConfig struct {
@@ -326,8 +386,8 @@ type ConsulGateway struct {
// Terminating represents the Consul Configuration Entry for a Terminating Gateway.
Terminating *ConsulTerminatingConfigEntry `hcl:"terminating,block"`
- // Mesh is not yet supported.
- // Mesh *ConsulMeshConfigEntry
+ // Mesh indicates the Consul service should be a Mesh Gateway.
+ Mesh *ConsulMeshConfigEntry `hcl:"mesh,block"`
}
func (g *ConsulGateway) Canonicalize() {
@@ -643,6 +703,21 @@ func (e *ConsulTerminatingConfigEntry) Copy() *ConsulTerminatingConfigEntry {
}
}
-// ConsulMeshConfigEntry is not yet supported.
-// type ConsulMeshConfigEntry struct {
-// }
+// ConsulMeshConfigEntry is a stub used to represent that the gateway service type
+// should be for a Mesh Gateway. Unlike Ingress and Terminating, there is no
+// actual Consul Config Entry type for mesh-gateway, at least for now. We still
+// create a type for future proofing, instead just using a bool for example.
+type ConsulMeshConfigEntry struct {
+ // nothing in here
+}
+
+func (e *ConsulMeshConfigEntry) Canonicalize() {
+ return
+}
+
+func (e *ConsulMeshConfigEntry) Copy() *ConsulMeshConfigEntry {
+ if e == nil {
+ return nil
+ }
+ return new(ConsulMeshConfigEntry)
+}
diff --git a/website/content/docs/job-specification/gateway.mdx b/website/content/docs/job-specification/gateway.mdx
index a679b805f88..54363a76c8e 100644
--- a/website/content/docs/job-specification/gateway.mdx
+++ b/website/content/docs/job-specification/gateway.mdx
@@ -43,6 +43,8 @@ Exactly one of `ingress` or `terminating` must be configured.
that will be associated with the service.
- `terminating` ([terminating]: nil)
- Configuration Entry of type `terminating-gateway`
that will be associated with the service.
+- `mesh` ([mesh]: nil)
- Indicates a mesh gateway will be associated
+ with the service.
### `proxy` Parameters
@@ -166,6 +168,10 @@ envoy_gateway_bind_addresses "" {
- `sni` `(string: )` - An optional hostname or domain name to specify during
the TLS handshake.
+### `mesh` Parameters
+
+ The `mesh` block currently does not have any configurable parameters.
+
### Gateway with host networking
Nomad supports running gateways using host networking. A static port must be allocated
@@ -433,6 +439,201 @@ job "countdash-terminating" {
}
```
+#### mesh gateway
+
+Mesh gateways are useful when Connect services need to make cross-datacenter
+requests where not all nodes in each datacenter have full connectivity. This example
+demonstrates using mesh gateways to enable making requests between datacenters
+`one` and `two`, where each mesh gateway will bind to the `public` host network
+configured on at least one Nomad client in each datacenter.
+
+Job running where Nomad and Consul are in datacenter `one`.
+
+```hcl
+job "countdash-mesh-one" {
+ datacenters = ["one"]
+
+ group "mesh-gateway-one" {
+ network {
+ mode = "bridge"
+
+ # A mesh gateway will require a host_network configured on at least one
+ # Nomad client that can establish cross-datacenter connections. Nomad will
+ # automatically schedule the mesh gateway task on compatible Nomad clients.
+ port "mesh_wan" {
+ host_network = "public"
+ }
+ }
+
+ service {
+ name = "mesh-gateway"
+
+ # The mesh gateway connect service should be configured to use a port from
+ # the host_network capable of cross-datacenter connections.
+ port = "mesh_wan"
+
+ connect {
+ gateway {
+ mesh {
+ # No configuration options in the mesh block.
+ }
+
+ # Consul gateway [envoy] proxy options.
+ proxy {
+ # The following options are automatically set by Nomad if not explicitly
+ # configured with using bridge networking.
+ #
+ # envoy_gateway_no_default_bind = true
+ # envoy_gateway_bind_addresses "lan" {
+ # address = "0.0.0.0"
+ # port =
+ # }
+ # envoy_gateway_bind_addresses "wan" {
+ # address = "0.0.0.0"
+ # port =
+ # }
+ # Additional options are documented at
+ # https://www.nomadproject.io/docs/job-specification/gateway#proxy-parameters
+ }
+ }
+ }
+ }
+ }
+
+ group "dashboard" {
+ network {
+ mode = "bridge"
+
+ port "http" {
+ static = 9002
+ to = 9002
+ }
+ }
+
+ service {
+ name = "count-dashboard"
+ port = "9002"
+
+ connect {
+ sidecar_service {
+ proxy {
+ upstreams {
+ destination_name = "count-api"
+ local_bind_port = 8080
+
+ # This dashboard service is running in datacenter "one", and will
+ # make requests to the "count-api" service running in datacenter
+ # "two", by going through the mesh gateway in each datacenter.
+ datacenter = "two"
+
+ mesh_gateway {
+ # Using "local" mode indicates requests should exit this datacenter
+ # through the mesh gateway, and enter the destination datacenter
+ # through a mesh gateway in that datacenter.
+ # Using "remote" mode indicates requests should bypass the local
+ # mesh gateway, instead directly connecting to the mesh gateway
+ # in the destination datacenter.
+ mode = "local"
+ }
+ }
+ }
+ }
+ }
+ }
+
+ task "dashboard" {
+ driver = "docker"
+
+ env {
+ COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
+ }
+
+ config {
+ image = "hashicorpnomad/counter-dashboard:v3"
+ }
+ }
+ }
+}
+```
+
+Job running where Nomad and Consul are in datacenter `two`.
+
+```hcl
+job "countdash-mesh-two" {
+ datacenters = ["two"]
+
+ group "mesh-gateway-two" {
+ network {
+ mode = "bridge"
+
+ # A mesh gateway will require a host_network configured for at least one
+ # Nomad client that can establish cross-datacenter connections. Nomad will
+ # automatically schedule the mesh gateway task on compatible Nomad clients.
+ port "mesh_wan" {
+ host_network = "public"
+ }
+ }
+
+ service {
+ name = "mesh-gateway"
+
+ # The mesh gateway connect service should be configured to use a port from
+ # the host_network capable of cross-datacenter connections.
+ port = "mesh_wan"
+
+ connect {
+ gateway {
+ mesh {
+ # No configuration options in the mesh block.
+ }
+
+ # Consul gateway [envoy] proxy options.
+ proxy {
+ # The following options are automatically set by Nomad if not explicitly
+ # configured with using bridge networking.
+ #
+ # envoy_gateway_no_default_bind = true
+ # envoy_gateway_bind_addresses "lan" {
+ # address = "0.0.0.0"
+ # port =
+ # }
+ # envoy_gateway_bind_addresses "wan" {
+ # address = "0.0.0.0"
+ # port =
+ # }
+ # Additional options are documented at
+ # https://www.nomadproject.io/docs/job-specification/gateway#proxy-parameters
+ }
+ }
+ }
+ }
+ }
+
+ group "api" {
+ network {
+ mode = "bridge"
+ }
+
+ service {
+ name = "count-api"
+ port = "9001"
+ connect {
+ sidecar_service {}
+ }
+ }
+
+ task "api" {
+ driver = "docker"
+
+ config {
+ image = "hashicorpnomad/counter-api:v3"
+ }
+ }
+ }
+}
+```
+
+
[address]: /docs/job-specification/gateway#address-parameters
[advanced configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
[connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout
@@ -446,3 +647,4 @@ job "countdash-terminating" {
[sidecar_task]: /docs/job-specification/sidecar_task
[terminating]: /docs/job-specification/gateway#terminating-parameters
[tls]: /docs/job-specification/gateway#tls-parameters
+[mesh]: /docs/job-specification/gateway#mesh-parameters
diff --git a/website/content/docs/job-specification/upstreams.mdx b/website/content/docs/job-specification/upstreams.mdx
index 5a8d27dc685..91bb9bd6cdc 100644
--- a/website/content/docs/job-specification/upstreams.mdx
+++ b/website/content/docs/job-specification/upstreams.mdx
@@ -52,8 +52,11 @@ job "countdash" {
upstreams {
destination_name = "count-api"
local_bind_port = 8080
- datacenter = "dc1"
+ datacenter = "dc2"
local_bind_address = "127.0.0.1"
+ mesh_gateway {
+ mode = "local"
+ }
}
}
}
@@ -86,6 +89,22 @@ job "countdash" {
local Consul datacenter.
- `local_bind_address` - `(string: "")` - The address the proxy will receive
connections for the upstream on.
+- `mesh_gateway` ([mesh_gateway][mesh_gateway_param]: nil)
- Configures the mesh gateway
+ behavior for connecting to this upstream.
+
+### `mesh_gateway` Parameters
+
+- `mode` `(string: "")` - The mode of operation in which to use [Connect Mesh Gateways][mesh_gateways].
+ If left unset, the mode will default to the mode as determined by the Consul [service-defaults][service_defaults_mode]
+ configuration for the service. Can be configured with the following modes:
+ - `local` - In this mode the Connect proxy makes its outbound connection to a
+ gateway running in the same datacenter. That gateway is then responsible for
+ ensuring the data gets forwarded along to gateways in the destination datacenter.
+ - `remote` - In this mode the Connect proxy makes its outbound connection to a
+ gateway running in the destination datacenter. That gateway will then forward
+ the data to the final destination service.
+ - `none` - In this mode, no gateway is used and a Connect proxy makes its
+ outbound connections directly to the destination services.
The `NOMAD_UPSTREAM_ADDR_` environment variables may be used
to interpolate the upstream's `host:port` address.
@@ -113,3 +132,6 @@ and a local bind port.
[interpolation]: /docs/runtime/interpolation 'Nomad interpolation'
[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
[upstreams]: /docs/job-specification/upstreams 'Nomad upstream config Specification'
+[service_defaults_mode]: https://www.consul.io/docs/connect/config-entries/service-defaults#meshgateway
+[mesh_gateway_param]: /docs/job-specification/upstreams#mesh_gateway-parameters
+[mesh_gateways]: https://www.consul.io/docs/connect/gateways/mesh-gateway#mesh-gateways