From 0479167252d955b78f289d65c830cbf94484851b Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 3 Jun 2021 10:28:48 -0500 Subject: [PATCH 1/4] client/fingerprint/consul: refactor the consul fingerprinter to test individual attributes This PR refactors the ConsulFingerprint implementation, breaking individual attributes into individual functions to make testing them easier. This is in preparation for additional extractors about to be added. Behavior should be otherwise unchanged. It adds the attribute consul.sku, which can be used to differentiate between Consul OSS vs Consul ENT. --- client/fingerprint/consul.go | 191 ++++--- client/fingerprint/consul_test.go | 503 +++++++++++------- .../test_fixtures/consul/agent_self.json | 478 +++++++++++++++++ 3 files changed, 920 insertions(+), 252 deletions(-) create mode 100644 client/fingerprint/test_fixtures/consul/agent_self.json diff --git a/client/fingerprint/consul.go b/client/fingerprint/consul.go index 3b2fe405489..183f39a79d0 100644 --- a/client/fingerprint/consul.go +++ b/client/fingerprint/consul.go @@ -3,10 +3,12 @@ package fingerprint import ( "fmt" "strconv" + "strings" "time" consul "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-version" ) const ( @@ -16,78 +18,120 @@ const ( // ConsulFingerprint is used to fingerprint for Consul type ConsulFingerprint struct { - logger log.Logger - client *consul.Client - lastState string + logger log.Logger + client *consul.Client + lastState string + extractors map[string]consulExtractor } +// consulInfo aliases the type returned from the Consul agent self endpoint. +type consulInfo = map[string]map[string]interface{} + +// consulExtractor is used to parse out one attribute from consulInfo. Returns +// the value of the attribute, and whether the attribute exists. +type consulExtractor func(consulInfo) (string, bool) + // NewConsulFingerprint is used to create a Consul fingerprint func NewConsulFingerprint(logger log.Logger) Fingerprint { - return &ConsulFingerprint{logger: logger.Named("consul"), lastState: consulUnavailable} + return &ConsulFingerprint{ + logger: logger.Named("consul"), + lastState: consulUnavailable, + } } func (f *ConsulFingerprint) Fingerprint(req *FingerprintRequest, resp *FingerprintResponse) error { - // Only create the client once to avoid creating too many connections to - // Consul. + + // establish consul client if necessary + if err := f.initialize(req); err != nil { + return err + } + + // query consul for agent self api + info := f.query(resp) + if len(info) == 0 { + // unable to reach consul, nothing to do this time + return nil + } + + // apply the extractor for each attribute + for attr, extractor := range f.extractors { + if s, ok := extractor(info); !ok { + f.logger.Warn("unable to fingerprint consul", "attribute", attr) + } else { + resp.AddAttribute(attr, s) + } + } + + // create link for consul + f.link(resp) + + // indicate Consul is now available + if f.lastState == consulUnavailable { + f.logger.Info("consul agent is available") + } + + f.lastState = consulAvailable + resp.Detected = true + return nil +} + +func (f *ConsulFingerprint) Periodic() (bool, time.Duration) { + return true, 15 * time.Second +} + +// clearConsulAttributes removes consul attributes and links from the passed Node. +func (f *ConsulFingerprint) clearConsulAttributes(r *FingerprintResponse) { + for attr := range f.extractors { + r.RemoveAttribute(attr) + } + r.RemoveLink("consul") +} + +func (f *ConsulFingerprint) initialize(req *FingerprintRequest) error { + // Only create the Consul client once to avoid creating many connections if f.client == nil { consulConfig, err := req.Config.ConsulConfig.ApiConfig() if err != nil { - return fmt.Errorf("Failed to initialize the Consul client config: %v", err) + return fmt.Errorf("failed to initialize Consul client config: %v", err) } f.client, err = consul.NewClient(consulConfig) if err != nil { - return fmt.Errorf("Failed to initialize consul client: %s", err) + return fmt.Errorf("failed to initialize Consul client: %s", err) + } + + f.extractors = map[string]consulExtractor{ + "consul.server": f.server, + "consul.version": f.version, + "consul.sku": f.sku, + "consul.revision": f.revision, + "unique.consul.name": f.name, + "consul.datacenter": f.dc, + "consul.segment": f.segment, } } + return nil +} + +func (f *ConsulFingerprint) query(resp *FingerprintResponse) consulInfo { // We'll try to detect consul by making a query to to the agent's self API. // If we can't hit this URL consul is probably not running on this machine. info, err := f.client.Agent().Self() if err != nil { f.clearConsulAttributes(resp) - // Print a message indicating that the Consul Agent is not available - // anymore + // indicate consul no longer available if f.lastState == consulAvailable { f.logger.Info("consul agent is unavailable") } f.lastState = consulUnavailable return nil } + return info +} - if s, ok := info["Config"]["Server"].(bool); ok { - resp.AddAttribute("consul.server", strconv.FormatBool(s)) - } else { - f.logger.Warn("unable to fingerprint consul.server") - } - if v, ok := info["Config"]["Version"].(string); ok { - resp.AddAttribute("consul.version", v) - } else { - f.logger.Warn("unable to fingerprint consul.version") - } - if r, ok := info["Config"]["Revision"].(string); ok { - resp.AddAttribute("consul.revision", r) - } else { - f.logger.Warn("unable to fingerprint consul.revision") - } - if n, ok := info["Config"]["NodeName"].(string); ok { - resp.AddAttribute("unique.consul.name", n) - } else { - f.logger.Warn("unable to fingerprint unique.consul.name") - } - if d, ok := info["Config"]["Datacenter"].(string); ok { - resp.AddAttribute("consul.datacenter", d) - } else { - f.logger.Warn("unable to fingerprint consul.datacenter") - } - if g, ok := info["Member"]["Tags"].(map[string]interface{}); ok { - if s, ok := g["segment"].(string); ok { - resp.AddAttribute("consul.segment", s) - } - } else { - f.logger.Warn("unable to fingerprint consul.segment") - } +func (f *ConsulFingerprint) link(resp *FingerprintResponse) { if dc, ok := resp.Attributes["consul.datacenter"]; ok { if name, ok2 := resp.Attributes["unique.consul.name"]; ok2 { resp.AddLink("consul", fmt.Sprintf("%s.%s", dc, name)) @@ -95,29 +139,54 @@ func (f *ConsulFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri } else { f.logger.Warn("malformed Consul response prevented linking") } +} - // If the Consul Agent was previously unavailable print a message to - // indicate the Agent is available now - if f.lastState == consulUnavailable { - f.logger.Info("consul agent is available") +func (f *ConsulFingerprint) server(info consulInfo) (string, bool) { + s, ok := info["Config"]["Server"].(bool) + return strconv.FormatBool(s), ok +} + +func (f *ConsulFingerprint) version(info consulInfo) (string, bool) { + v, ok := info["Config"]["Version"].(string) + return v, ok +} + +func (f *ConsulFingerprint) sku(info consulInfo) (string, bool) { + v, ok := info["Config"]["Version"].(string) + if !ok { + return "", ok } - f.lastState = consulAvailable - resp.Detected = true - return nil + + ver, vErr := version.NewVersion(v) + if vErr != nil { + return "", false + } + if strings.Contains(ver.Metadata(), "ent") { + return "ent", true + } + return "oss", true } -// clearConsulAttributes removes consul attributes and links from the passed -// Node. -func (f *ConsulFingerprint) clearConsulAttributes(r *FingerprintResponse) { - r.RemoveAttribute("consul.server") - r.RemoveAttribute("consul.version") - r.RemoveAttribute("consul.revision") - r.RemoveAttribute("unique.consul.name") - r.RemoveAttribute("consul.datacenter") - r.RemoveAttribute("consul.segment") - r.RemoveLink("consul") +func (f *ConsulFingerprint) revision(info consulInfo) (string, bool) { + r, ok := info["Config"]["Revision"].(string) + return r, ok } -func (f *ConsulFingerprint) Periodic() (bool, time.Duration) { - return true, 15 * time.Second +func (f *ConsulFingerprint) name(info consulInfo) (string, bool) { + n, ok := info["Config"]["NodeName"].(string) + return n, ok +} + +func (f *ConsulFingerprint) dc(info consulInfo) (string, bool) { + d, ok := info["Config"]["Datacenter"].(string) + return d, ok +} + +func (f *ConsulFingerprint) segment(info consulInfo) (string, bool) { + tags, tagsOK := info["Member"]["Tags"].(map[string]interface{}) + if !tagsOK { + return "", false + } + s, ok := tags["segment"].(string) + return s, ok } diff --git a/client/fingerprint/consul_test.go b/client/fingerprint/consul_test.go index 4f0287fbfbc..5e3c40930b6 100644 --- a/client/fingerprint/consul_test.go +++ b/client/fingerprint/consul_test.go @@ -1,7 +1,8 @@ package fingerprint import ( - "fmt" + "io" + "io/ioutil" "net/http" "net/http/httptest" "strings" @@ -10,207 +11,327 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/structs" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestConsulFingerprint(t *testing.T) { - fp := NewConsulFingerprint(testlog.HCLogger(t)) - node := &structs.Node{ - Attributes: make(map[string]string), - } +// fakeConsul creates an HTTP server mimicking Consul /v1/agent/self endpoint on +// the first request, and alternates between success and failure responses on +// subsequent requests +func fakeConsul(payload string) (*httptest.Server, *config.Config) { + working := true ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - fmt.Fprintln(w, mockConsulResponse) + if working { + _, _ = io.WriteString(w, payload) + working = false + } else { + w.WriteHeader(http.StatusInternalServerError) + working = true + } })) - defer ts.Close() - conf := config.DefaultConfig() - conf.ConsulConfig.Addr = strings.TrimPrefix(ts.URL, "http://") - - request := &FingerprintRequest{Config: conf, Node: node} - var response FingerprintResponse - err := fp.Fingerprint(request, &response) - if err != nil { - t.Fatalf("Failed to fingerprint: %s", err) - } - - if !response.Detected { - t.Fatalf("expected response to be applicable") - } - - assertNodeAttributeContains(t, response.Attributes, "consul.server") - assertNodeAttributeContains(t, response.Attributes, "consul.version") - assertNodeAttributeContains(t, response.Attributes, "consul.revision") - assertNodeAttributeContains(t, response.Attributes, "unique.consul.name") - assertNodeAttributeContains(t, response.Attributes, "consul.datacenter") - assertNodeAttributeContains(t, response.Attributes, "consul.segment") - - if _, ok := response.Links["consul"]; !ok { - t.Errorf("Expected a link to consul, none found") - } + cfg := config.DefaultConfig() + cfg.ConsulConfig.Addr = strings.TrimPrefix(ts.URL, `http://`) + return ts, cfg } -// Taken from tryconsul using consul release 0.5.2 -const mockConsulResponse = ` -{ - "Config": { - "Bootstrap": false, - "BootstrapExpect": 3, - "Server": true, - "Datacenter": "vagrant", - "DataDir": "/var/lib/consul", - "DNSRecursor": "", - "DNSRecursors": [], - "DNSConfig": { - "NodeTTL": 0, - "ServiceTTL": null, - "AllowStale": false, - "EnableTruncate": false, - "MaxStale": 5000000000, - "OnlyPassing": false - }, - "Domain": "consul.", - "LogLevel": "INFO", - "NodeName": "consul2", - "ClientAddr": "0.0.0.0", - "BindAddr": "0.0.0.0", - "AdvertiseAddr": "172.16.59.133", - "AdvertiseAddrWan": "172.16.59.133", - "Ports": { - "DNS": 8600, - "HTTP": 8500, - "HTTPS": -1, - "RPC": 8400, - "SerfLan": 8301, - "SerfWan": 8302, - "Server": 8300 - }, - "Addresses": { - "DNS": "", - "HTTP": "", - "HTTPS": "", - "RPC": "" - }, - "LeaveOnTerm": false, - "SkipLeaveOnInt": false, - "StatsiteAddr": "", - "StatsitePrefix": "consul", - "StatsdAddr": "", - "Protocol": 2, - "EnableDebug": false, - "VerifyIncoming": false, - "VerifyOutgoing": false, - "VerifyServerHostname": false, - "CAFile": "", - "CertFile": "", - "KeyFile": "", - "ServerName": "", - "StartJoin": [], - "StartJoinWan": [], - "RetryJoin": [], - "RetryMaxAttempts": 0, - "RetryIntervalRaw": "", - "RetryJoinWan": [], - "RetryMaxAttemptsWan": 0, - "RetryIntervalWanRaw": "", - "UiDir": "/opt/consul-ui", - "PidFile": "", - "EnableSyslog": true, - "SyslogFacility": "LOCAL0", - "RejoinAfterLeave": false, - "CheckUpdateInterval": 300000000000, - "ACLDatacenter": "", - "ACLTTL": 30000000000, - "ACLTTLRaw": "", - "ACLDefaultPolicy": "allow", - "ACLDownPolicy": "extend-cache", - "Watches": null, - "DisableRemoteExec": false, - "DisableUpdateCheck": false, - "DisableAnonymousSignature": false, - "HTTPAPIResponseHeaders": null, - "AtlasInfrastructure": "", - "AtlasJoin": false, - "Revision": "9a9cc9341bb487651a0399e3fc5e1e8a42e62dd9+CHANGES", - "Version": "0.5.2", - "VersionPrerelease": "", - "UnixSockets": { - "Usr": "", - "Grp": "", - "Perms": "" - }, - "SessionTTLMin": 0, - "SessionTTLMinRaw": "" - }, - "Member": { - "Name": "consul2", - "Addr": "172.16.59.133", - "Port": 8301, - "Tags": { - "build": "0.5.2:9a9cc934", - "dc": "vagrant", - "expect": "3", - "port": "8300", - "role": "consul", - "segment": "mysegment", - "vsn": "2" - }, - "Status": 1, - "ProtocolMin": 1, - "ProtocolMax": 2, - "ProtocolCur": 2, - "DelegateMin": 2, - "DelegateMax": 4, - "DelegateCur": 4 - } +func fakeConsulPayload(t *testing.T, filename string) string { + b, err := ioutil.ReadFile(filename) + require.NoError(t, err) + return string(b) } -` - -// TestConsulFingerprint_UnexpectedResponse asserts that the Consul -// fingerprinter does not panic when it encounters an unexpected response. -// See https://github.com/hashicorp/nomad/issues/3326 -func TestConsulFingerprint_UnexpectedResponse(t *testing.T) { - assert := assert.New(t) - fp := NewConsulFingerprint(testlog.HCLogger(t)) - node := &structs.Node{ - Attributes: make(map[string]string), - } - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - fmt.Fprintln(w, "{}") - })) +func newConsulFingerPrint(t *testing.T) *ConsulFingerprint { + return NewConsulFingerprint(testlog.HCLogger(t)).(*ConsulFingerprint) +} + +func TestConsulFingerprint_server(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("is server", func(t *testing.T) { + s, ok := fp.server(consulInfo{ + "Config": {"Server": true}, + }) + require.True(t, ok) + require.Equal(t, "true", s) + }) + + t.Run("is not server", func(t *testing.T) { + s, ok := fp.server(consulInfo{ + "Config": {"Server": false}, + }) + require.True(t, ok) + require.Equal(t, "false", s) + }) + + t.Run("missing", func(t *testing.T) { + _, ok := fp.server(consulInfo{ + "Config": {}, + }) + require.False(t, ok) + }) + + t.Run("malformed", func(t *testing.T) { + _, ok := fp.server(consulInfo{ + "Config": {"Server": 9000}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_version(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("oss", func(t *testing.T) { + v, ok := fp.version(consulInfo{ + "Config": {"Version": "v1.9.5"}, + }) + require.True(t, ok) + require.Equal(t, "v1.9.5", v) + }) + + t.Run("ent", func(t *testing.T) { + v, ok := fp.version(consulInfo{ + "Config": {"Version": "v1.9.5+ent"}, + }) + require.True(t, ok) + require.Equal(t, "v1.9.5+ent", v) + }) + + t.Run("missing", func(t *testing.T) { + _, ok := fp.version(consulInfo{ + "Config": {}, + }) + require.False(t, ok) + }) + + t.Run("malformed", func(t *testing.T) { + _, ok := fp.version(consulInfo{ + "Config": {"Version": 9000}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_sku(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("oss", func(t *testing.T) { + s, ok := fp.sku(consulInfo{ + "Config": {"Version": "v1.9.5"}, + }) + require.True(t, ok) + require.Equal(t, "oss", s) + }) + + t.Run("oss dev", func(t *testing.T) { + s, ok := fp.sku(consulInfo{ + "Config": {"Version": "v1.9.5-dev"}, + }) + require.True(t, ok) + require.Equal(t, "oss", s) + }) + + t.Run("ent", func(t *testing.T) { + s, ok := fp.sku(consulInfo{ + "Config": {"Version": "v1.9.5+ent"}, + }) + require.True(t, ok) + require.Equal(t, "ent", s) + }) + + t.Run("ent dev", func(t *testing.T) { + s, ok := fp.sku(consulInfo{ + "Config": {"Version": "v1.9.5+ent-dev"}, + }) + require.True(t, ok) + require.Equal(t, "ent", s) + }) + + t.Run("missing", func(t *testing.T) { + _, ok := fp.sku(consulInfo{ + "Config": {}, + }) + require.False(t, ok) + }) + + t.Run("malformed", func(t *testing.T) { + _, ok := fp.sku(consulInfo{ + "Config": {"Version": "***"}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_revision(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("ok", func(t *testing.T) { + r, ok := fp.revision(consulInfo{ + "Config": {"Revision": "3c1c22679"}, + }) + require.True(t, ok) + require.Equal(t, "3c1c22679", r) + }) + + t.Run("malformed", func(t *testing.T) { + _, ok := fp.revision(consulInfo{ + "Config": {"Revision": 9000}, + }) + require.False(t, ok) + }) + + t.Run("missing", func(t *testing.T) { + _, ok := fp.revision(consulInfo{ + "Config": {}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_dc(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("ok", func(t *testing.T) { + dc, ok := fp.dc(consulInfo{ + "Config": {"Datacenter": "dc1"}, + }) + require.True(t, ok) + require.Equal(t, "dc1", dc) + }) + + t.Run("malformed", func(t *testing.T) { + _, ok := fp.dc(consulInfo{ + "Config": {"Datacenter": 9000}, + }) + require.False(t, ok) + }) + + t.Run("missing", func(t *testing.T) { + _, ok := fp.dc(consulInfo{ + "Config": {}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_segment(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("ok", func(t *testing.T) { + s, ok := fp.segment(consulInfo{ + "Member": {"Tags": map[string]interface{}{"segment": "seg1"}}, + }) + require.True(t, ok) + require.Equal(t, "seg1", s) + }) + + t.Run("segment missing", func(t *testing.T) { + _, ok := fp.segment(consulInfo{ + "Member": {"Tags": map[string]interface{}{}}, + }) + require.False(t, ok) + }) + + t.Run("tags missing", func(t *testing.T) { + _, ok := fp.segment(consulInfo{ + "Member": {}, + }) + require.False(t, ok) + }) + + t.Run("malformed", func(t *testing.T) { + _, ok := fp.segment(consulInfo{ + "Member": {"Tags": map[string]interface{}{"segment": 9000}}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_Fingerprint(t *testing.T) { + cf := newConsulFingerPrint(t) + + ts, cfg := fakeConsul(fakeConsulPayload(t, "test_fixtures/consul/agent_self.json")) defer ts.Close() - conf := config.DefaultConfig() - conf.ConsulConfig.Addr = strings.TrimPrefix(ts.URL, "http://") - - request := &FingerprintRequest{Config: conf, Node: node} - var response FingerprintResponse - err := fp.Fingerprint(request, &response) - assert.Nil(err) - - if !response.Detected { - t.Fatalf("expected response to be applicable") - } - - attrs := []string{ - "consul.server", - "consul.version", - "consul.revision", - "unique.consul.name", - "consul.datacenter", - "consul.segment", - } - - for _, attr := range attrs { - if v, ok := response.Attributes[attr]; ok { - t.Errorf("unexpected node attribute %q with vlaue %q", attr, v) - } - } + node := &structs.Node{Attributes: make(map[string]string)} + + // consul not available before first run + require.Equal(t, consulUnavailable, cf.lastState) + + // execute first query with good response + var resp FingerprintResponse + err := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp) + require.NoError(t, err) + require.Equal(t, map[string]string{ + "consul.datacenter": "dc1", + "consul.revision": "3c1c22679", + "consul.segment": "seg1", + "consul.server": "true", + "consul.sku": "oss", + "consul.version": "1.9.5", + "unique.consul.name": "HAL9000", + }, resp.Attributes) + require.True(t, resp.Detected) + + // consul now available + require.Equal(t, consulAvailable, cf.lastState) + + var resp2 FingerprintResponse + + // pretend attributes set for failing request + node.Attributes["consul.datacenter"] = "foo" + node.Attributes["consul.revision"] = "foo" + node.Attributes["consul.segment"] = "foo" + node.Attributes["consul.server"] = "foo" + node.Attributes["consul.sku"] = "foo" + node.Attributes["consul.version"] = "foo" + node.Attributes["unique.consul.name"] = "foo" + + // execute second query with error + err2 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp2) + require.NoError(t, err2) // does not return error + require.Equal(t, map[string]string{ // attributes set empty + "consul.datacenter": "", + "consul.revision": "", + "consul.segment": "", + "consul.server": "", + "consul.sku": "", + "consul.version": "", + "unique.consul.name": "", + }, resp2.Attributes) + require.True(t, resp.Detected) // never downgrade + + // consul no longer available + require.Equal(t, consulUnavailable, cf.lastState) + + // execute third query no error + var resp3 FingerprintResponse + err3 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp3) + require.NoError(t, err3) + require.Equal(t, map[string]string{ + "consul.datacenter": "dc1", + "consul.revision": "3c1c22679", + "consul.segment": "seg1", + "consul.server": "true", + "consul.sku": "oss", + "consul.version": "1.9.5", + "unique.consul.name": "HAL9000", + }, resp3.Attributes) - if v, ok := response.Links["consul"]; ok { - t.Errorf("Unexpected link to consul: %v", v) - } + // consul now available again + require.Equal(t, consulAvailable, cf.lastState) + require.True(t, resp.Detected) } diff --git a/client/fingerprint/test_fixtures/consul/agent_self.json b/client/fingerprint/test_fixtures/consul/agent_self.json new file mode 100644 index 00000000000..e1698a12bcf --- /dev/null +++ b/client/fingerprint/test_fixtures/consul/agent_self.json @@ -0,0 +1,478 @@ +{ + "Config": { + "Datacenter": "dc1", + "NodeName": "HAL9000", + "NodeID": "fa512dd3-4e92-6fb5-6446-bd7ed012ebe0", + "Revision": "3c1c22679", + "Server": true, + "Version": "1.9.5" + }, + "DebugConfig": { + "ACLDatacenter": "dc1", + "ACLDefaultPolicy": "allow", + "ACLDisabledTTL": "2m0s", + "ACLDownPolicy": "extend-cache", + "ACLEnableKeyListPolicy": false, + "ACLMasterToken": "hidden", + "ACLPolicyTTL": "30s", + "ACLRoleTTL": "0s", + "ACLTokenReplication": false, + "ACLTokenTTL": "30s", + "ACLTokens": { + "ACLAgentMasterToken": "hidden", + "ACLAgentToken": "hidden", + "ACLDefaultToken": "hidden", + "ACLReplicationToken": "hidden", + "DataDir": "", + "EnablePersistence": false, + "EnterpriseConfig": {} + }, + "ACLsEnabled": false, + "AEInterval": "1m0s", + "AdvertiseAddrLAN": "127.0.0.1", + "AdvertiseAddrWAN": "127.0.0.1", + "AdvertiseReconnectTimeout": "0s", + "AllowWriteHTTPFrom": [], + "AutoConfig": { + "Authorizer": { + "AllowReuse": false, + "AuthMethod": { + "ACLAuthMethodEnterpriseFields": {}, + "Config": { + "BoundAudiences": null, + "BoundIssuer": "", + "ClaimMappings": null, + "ClockSkewLeeway": 0, + "ExpirationLeeway": 0, + "JWKSCACert": "", + "JWKSURL": "", + "JWTSupportedAlgs": null, + "JWTValidationPubKeys": null, + "ListClaimMappings": null, + "NotBeforeLeeway": 0, + "OIDCDiscoveryCACert": "", + "OIDCDiscoveryURL": "" + }, + "Description": "", + "DisplayName": "", + "EnterpriseMeta": {}, + "MaxTokenTTL": "0s", + "Name": "Auto Config Authorizer", + "RaftIndex": { + "CreateIndex": 0, + "ModifyIndex": 0 + }, + "TokenLocality": "", + "Type": "jwt" + }, + "ClaimAssertions": [], + "Enabled": false + }, + "DNSSANs": [], + "Enabled": false, + "IPSANs": [], + "IntroToken": "hidden", + "IntroTokenFile": "", + "ServerAddresses": [] + }, + "AutoEncryptAllowTLS": false, + "AutoEncryptDNSSAN": [], + "AutoEncryptIPSAN": [], + "AutoEncryptTLS": false, + "AutopilotCleanupDeadServers": true, + "AutopilotDisableUpgradeMigration": false, + "AutopilotLastContactThreshold": "200ms", + "AutopilotMaxTrailingLogs": 250, + "AutopilotMinQuorum": 0, + "AutopilotRedundancyZoneTag": "", + "AutopilotServerStabilizationTime": "10s", + "AutopilotUpgradeVersionTag": "", + "BindAddr": "127.0.0.1", + "Bootstrap": false, + "BootstrapExpect": 0, + "CAFile": "", + "CAPath": "", + "Cache": { + "EntryFetchMaxBurst": 2, + "EntryFetchRate": 1.7976931348623157e+308, + "Logger": null + }, + "CertFile": "", + "CheckDeregisterIntervalMin": "1m0s", + "CheckOutputMaxSize": 4096, + "CheckReapInterval": "30s", + "CheckUpdateInterval": "5m0s", + "Checks": [], + "ClientAddrs": [ + "127.0.0.1" + ], + "ConfigEntryBootstrap": [], + "ConnectCAConfig": {}, + "ConnectCAProvider": "", + "ConnectEnabled": true, + "ConnectMeshGatewayWANFederationEnabled": false, + "ConnectSidecarMaxPort": 21255, + "ConnectSidecarMinPort": 21000, + "ConnectTestCALeafRootChangeSpread": "0s", + "ConsulCoordinateUpdateBatchSize": 128, + "ConsulCoordinateUpdateMaxBatches": 5, + "ConsulCoordinateUpdatePeriod": "100ms", + "ConsulRaftElectionTimeout": "52ms", + "ConsulRaftHeartbeatTimeout": "35ms", + "ConsulRaftLeaderLeaseTimeout": "20ms", + "ConsulServerHealthInterval": "10ms", + "DNSARecordLimit": 0, + "DNSAddrs": [ + "tcp://127.0.0.1:8600", + "udp://127.0.0.1:8600" + ], + "DNSAllowStale": true, + "DNSAltDomain": "", + "DNSCacheMaxAge": "0s", + "DNSDisableCompression": false, + "DNSDomain": "consul.", + "DNSEnableTruncate": false, + "DNSMaxStale": "87600h0m0s", + "DNSNodeMetaTXT": true, + "DNSNodeTTL": "0s", + "DNSOnlyPassing": false, + "DNSPort": 8600, + "DNSRecursorTimeout": "2s", + "DNSRecursors": [], + "DNSSOA": { + "Expire": 86400, + "Minttl": 0, + "Refresh": 3600, + "Retry": 600 + }, + "DNSServiceTTL": {}, + "DNSUDPAnswerLimit": 3, + "DNSUseCache": false, + "DataDir": "", + "Datacenter": "dc1", + "DefaultQueryTime": "5m0s", + "DevMode": true, + "DisableAnonymousSignature": true, + "DisableCoordinates": false, + "DisableHTTPUnprintableCharFilter": false, + "DisableHostNodeID": true, + "DisableKeyringFile": true, + "DisableRemoteExec": true, + "DisableUpdateCheck": false, + "DiscardCheckOutput": false, + "DiscoveryMaxStale": "0s", + "EnableAgentTLSForChecks": false, + "EnableCentralServiceConfig": true, + "EnableDebug": true, + "EnableLocalScriptChecks": false, + "EnableRemoteScriptChecks": false, + "EncryptKey": "hidden", + "EncryptVerifyIncoming": true, + "EncryptVerifyOutgoing": true, + "EnterpriseRuntimeConfig": {}, + "ExposeMaxPort": 21755, + "ExposeMinPort": 21500, + "GRPCAddrs": [ + "tcp://127.0.0.1:8502" + ], + "GRPCPort": 8502, + "GossipLANGossipInterval": "100ms", + "GossipLANGossipNodes": 3, + "GossipLANProbeInterval": "100ms", + "GossipLANProbeTimeout": "100ms", + "GossipLANRetransmitMult": 4, + "GossipLANSuspicionMult": 3, + "GossipWANGossipInterval": "100ms", + "GossipWANGossipNodes": 3, + "GossipWANProbeInterval": "100ms", + "GossipWANProbeTimeout": "100ms", + "GossipWANRetransmitMult": 4, + "GossipWANSuspicionMult": 3, + "HTTPAddrs": [ + "tcp://127.0.0.1:8500" + ], + "HTTPBlockEndpoints": [], + "HTTPMaxConnsPerClient": 200, + "HTTPMaxHeaderBytes": 0, + "HTTPPort": 8500, + "HTTPResponseHeaders": {}, + "HTTPSAddrs": [], + "HTTPSHandshakeTimeout": "5s", + "HTTPSPort": -1, + "HTTPUseCache": true, + "KVMaxValueSize": 524288, + "KeyFile": "hidden", + "LeaveDrainTime": "5s", + "LeaveOnTerm": false, + "Logging": { + "EnableSyslog": false, + "LogFilePath": "", + "LogJSON": false, + "LogLevel": "DEBUG", + "LogRotateBytes": 0, + "LogRotateDuration": "0s", + "LogRotateMaxFiles": 0, + "Name": "", + "SyslogFacility": "LOCAL0" + }, + "MaxQueryTime": "10m0s", + "NodeID": "fa512dd3-4e92-6fb5-6446-bd7ed012ebe0", + "NodeMeta": {}, + "NodeName": "x52", + "PidFile": "", + "PrimaryDatacenter": "dc1", + "PrimaryGateways": [], + "PrimaryGatewaysInterval": "30s", + "RPCAdvertiseAddr": "tcp://127.0.0.1:8300", + "RPCBindAddr": "tcp://127.0.0.1:8300", + "RPCConfig": { + "EnableStreaming": false + }, + "RPCHandshakeTimeout": "5s", + "RPCHoldTimeout": "7s", + "RPCMaxBurst": 1000, + "RPCMaxConnsPerClient": 100, + "RPCProtocol": 2, + "RPCRateLimit": -1, + "RaftProtocol": 3, + "RaftSnapshotInterval": "0s", + "RaftSnapshotThreshold": 0, + "RaftTrailingLogs": 0, + "ReadReplica": false, + "ReconnectTimeoutLAN": "0s", + "ReconnectTimeoutWAN": "0s", + "RejoinAfterLeave": false, + "RetryJoinIntervalLAN": "30s", + "RetryJoinIntervalWAN": "30s", + "RetryJoinLAN": [], + "RetryJoinMaxAttemptsLAN": 0, + "RetryJoinMaxAttemptsWAN": 0, + "RetryJoinWAN": [], + "Revision": "", + "SegmentLimit": 64, + "SegmentName": "", + "SegmentNameLimit": 64, + "Segments": [], + "SerfAdvertiseAddrLAN": "tcp://127.0.0.1:8301", + "SerfAdvertiseAddrWAN": "tcp://127.0.0.1:8302", + "SerfAllowedCIDRsLAN": [], + "SerfAllowedCIDRsWAN": [], + "SerfBindAddrLAN": "tcp://127.0.0.1:8301", + "SerfBindAddrWAN": "tcp://127.0.0.1:8302", + "SerfPortLAN": 8301, + "SerfPortWAN": 8302, + "ServerMode": true, + "ServerName": "", + "ServerPort": 8300, + "Services": [], + "SessionTTLMin": "0s", + "SkipLeaveOnInt": true, + "StartJoinAddrsLAN": [], + "StartJoinAddrsWAN": [], + "SyncCoordinateIntervalMin": "15s", + "SyncCoordinateRateTarget": 64, + "TLSCipherSuites": [], + "TLSMinVersion": "tls12", + "TLSPreferServerCipherSuites": false, + "TaggedAddresses": { + "lan": "127.0.0.1", + "lan_ipv4": "127.0.0.1", + "wan": "127.0.0.1", + "wan_ipv4": "127.0.0.1" + }, + "Telemetry": { + "AllowedPrefixes": [], + "BlockedPrefixes": [], + "CirconusAPIApp": "", + "CirconusAPIToken": "hidden", + "CirconusAPIURL": "", + "CirconusBrokerID": "", + "CirconusBrokerSelectTag": "", + "CirconusCheckDisplayName": "", + "CirconusCheckForceMetricActivation": "", + "CirconusCheckID": "", + "CirconusCheckInstanceID": "", + "CirconusCheckSearchTag": "", + "CirconusCheckTags": "", + "CirconusSubmissionInterval": "", + "CirconusSubmissionURL": "", + "Disable": false, + "DisableCompatOneNine": false, + "DisableHostname": false, + "DogstatsdAddr": "", + "DogstatsdTags": [], + "FilterDefault": true, + "MetricsPrefix": "consul", + "PrometheusOpts": { + "CounterDefinitions": [], + "Expiration": "0s", + "GaugeDefinitions": [], + "Registerer": null, + "SummaryDefinitions": [] + }, + "StatsdAddr": "", + "StatsiteAddr": "" + }, + "TranslateWANAddrs": false, + "TxnMaxReqLen": 524288, + "UIConfig": { + "ContentPath": "/ui/", + "DashboardURLTemplates": {}, + "Dir": "", + "Enabled": true, + "MetricsProvider": "", + "MetricsProviderFiles": [], + "MetricsProviderOptionsJSON": "", + "MetricsProxy": { + "AddHeaders": [], + "BaseURL": "", + "PathAllowlist": [] + } + }, + "UnixSocketGroup": "", + "UnixSocketMode": "", + "UnixSocketUser": "", + "UseStreamingBackend": false, + "VerifyIncoming": false, + "VerifyIncomingHTTPS": false, + "VerifyIncomingRPC": false, + "VerifyOutgoing": false, + "VerifyServerHostname": false, + "Version": "1.9.5", + "VersionPrerelease": "", + "Watches": [] + }, + "Coord": { + "Vec": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "Error": 1.5, + "Adjustment": 0, + "Height": 0.00001 + }, + "Member": { + "Name": "x52", + "Addr": "127.0.0.1", + "Port": 8301, + "Tags": { + "acls": "0", + "build": "1.9.5:", + "dc": "dc1", + "ft_fs": "1", + "ft_si": "1", + "id": "fa512dd3-4e92-6fb5-6446-bd7ed012ebe0", + "port": "8300", + "raft_vsn": "3", + "role": "consul", + "segment": "seg1", + "vsn": "2", + "vsn_max": "3", + "vsn_min": "2", + "wan_join_port": "8302" + }, + "Status": 1, + "ProtocolMin": 1, + "ProtocolMax": 5, + "ProtocolCur": 2, + "DelegateMin": 2, + "DelegateMax": 5, + "DelegateCur": 4 + }, + "Stats": { + "agent": { + "check_monitors": "0", + "check_ttls": "0", + "checks": "0", + "services": "0" + }, + "build": { + "prerelease": "", + "revision": "", + "version": "1.9.5" + }, + "consul": { + "acl": "disabled", + "bootstrap": "false", + "known_datacenters": "1", + "leader": "true", + "leader_addr": "127.0.0.1:8300", + "server": "true" + }, + "raft": { + "applied_index": "13", + "commit_index": "13", + "fsm_pending": "0", + "last_contact": "0", + "last_log_index": "13", + "last_log_term": "2", + "last_snapshot_index": "0", + "last_snapshot_term": "0", + "latest_configuration": "[{Suffrage:Voter ID:fa512dd3-4e92-6fb5-6446-bd7ed012ebe0 Address:127.0.0.1:8300}]", + "latest_configuration_index": "0", + "num_peers": "0", + "protocol_version": "3", + "protocol_version_max": "3", + "protocol_version_min": "0", + "snapshot_version_max": "1", + "snapshot_version_min": "0", + "state": "Leader", + "term": "2" + }, + "runtime": { + "arch": "amd64", + "cpu_count": "24", + "goroutines": "85", + "max_procs": "24", + "os": "linux", + "version": "go1.16.4" + }, + "serf_lan": { + "coordinate_resets": "0", + "encrypted": "false", + "event_queue": "1", + "event_time": "2", + "failed": "0", + "health_score": "0", + "intent_queue": "0", + "left": "0", + "member_time": "1", + "members": "1", + "query_queue": "0", + "query_time": "1" + }, + "serf_wan": { + "coordinate_resets": "0", + "encrypted": "false", + "event_queue": "0", + "event_time": "1", + "failed": "0", + "health_score": "0", + "intent_queue": "0", + "left": "0", + "member_time": "1", + "members": "1", + "query_queue": "0", + "query_time": "1" + } + }, + "Meta": { + "consul-network-segment": "" + }, + "xDS": { + "SupportedProxies": { + "envoy": [ + "1.16.2", + "1.15.3", + "1.14.6", + "1.13.7" + ] + } + } +} From b3254f618af188ef1a028a9a2495ebb3544dcc91 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 3 Jun 2021 12:38:43 -0500 Subject: [PATCH 2/4] client/fingerprint/consul: add new attributes to consul fingerprinter This PR adds new probes for detecting these new Consul related attributes: Consul namespaces are a Consul enterprise feature that may be disabled depending on the enterprise license associated with the Consul servers. Having this attribute available will enable Nomad to properly decide whether to query the Consul Namespace API. Consul connect must be explicitly enabled before Connect APIs will work. Currently Nomad only checks for a minimum Consul version. Having this attribute available will enable Nomad to properly schedule Connect tasks only on nodes with a Consul agent that has Connect enabled. Consul connect requires the grpc port to be explicitly set before Connect APIs will work. Currently Nomad only checks for a minimal Consul version. Having this attribute available will enable Nomad to schedule Connect tasks only on nodes with a Consul agent that has the grpc listener enabled. --- client/fingerprint/consul.go | 53 +- client/fingerprint/consul_test.go | 210 +++++++- .../test_fixtures/consul/agent_self_ent.json | 502 ++++++++++++++++++ .../{agent_self.json => agent_self_oss.json} | 0 4 files changed, 749 insertions(+), 16 deletions(-) create mode 100644 client/fingerprint/test_fixtures/consul/agent_self_ent.json rename client/fingerprint/test_fixtures/consul/{agent_self.json => agent_self_oss.json} (100%) diff --git a/client/fingerprint/consul.go b/client/fingerprint/consul.go index 183f39a79d0..e37ffb1a888 100644 --- a/client/fingerprint/consul.go +++ b/client/fingerprint/consul.go @@ -101,13 +101,16 @@ func (f *ConsulFingerprint) initialize(req *FingerprintRequest) error { } f.extractors = map[string]consulExtractor{ - "consul.server": f.server, - "consul.version": f.version, - "consul.sku": f.sku, - "consul.revision": f.revision, - "unique.consul.name": f.name, - "consul.datacenter": f.dc, - "consul.segment": f.segment, + "consul.server": f.server, + "consul.version": f.version, + "consul.sku": f.sku, + "consul.revision": f.revision, + "unique.consul.name": f.name, + "consul.datacenter": f.dc, + "consul.segment": f.segment, + "consul.connect": f.connect, + "consul.grpc": f.grpc, + "consul.ft.namespaces": f.namespaces, } } @@ -190,3 +193,39 @@ func (f *ConsulFingerprint) segment(info consulInfo) (string, bool) { s, ok := tags["segment"].(string) return s, ok } + +func (f *ConsulFingerprint) connect(info consulInfo) (string, bool) { + c, ok := info["DebugConfig"]["ConnectEnabled"].(bool) + return strconv.FormatBool(c), ok +} + +func (f *ConsulFingerprint) grpc(info consulInfo) (string, bool) { + p, ok := info["DebugConfig"]["GRPCPort"].(float64) + return fmt.Sprintf("%d", int(p)), ok +} + +func (f *ConsulFingerprint) namespaces(info consulInfo) (string, bool) { + return f.feature("Namespaces", info) +} + +// possible values as of v1.9.5+ent: +// Automated Backups, Automated Upgrades, Enhanced Read Scalability, +// Network Segments, Redundancy Zone, Advanced Network Federation, +// Namespaces, SSO, Audit Logging +func (f *ConsulFingerprint) feature(name string, info consulInfo) (string, bool) { + lic, licOK := info["Stats"]["license"].(map[string]interface{}) + if !licOK { + return "", false + } + + features, exists := lic["features"].(string) + if !exists { + return "", false + } + + if !strings.Contains(features, name) { + return "", false + } + + return "true", true +} diff --git a/client/fingerprint/consul_test.go b/client/fingerprint/consul_test.go index 5e3c40930b6..e5eda50b292 100644 --- a/client/fingerprint/consul_test.go +++ b/client/fingerprint/consul_test.go @@ -260,10 +260,105 @@ func TestConsulFingerprint_segment(t *testing.T) { }) } -func TestConsulFingerprint_Fingerprint(t *testing.T) { +func TestConsulFingerprint_connect(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("connect enabled", func(t *testing.T) { + s, ok := fp.connect(consulInfo{ + "DebugConfig": {"ConnectEnabled": true}, + }) + require.True(t, ok) + require.Equal(t, "true", s) + }) + + t.Run("connect not enabled", func(t *testing.T) { + s, ok := fp.connect(consulInfo{ + "DebugConfig": {"ConnectEnabled": false}, + }) + require.True(t, ok) + require.Equal(t, "false", s) + }) + + t.Run("connect missing", func(t *testing.T) { + _, ok := fp.connect(consulInfo{ + "DebugConfig": {}, + }) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_grpc(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("grpc set", func(t *testing.T) { + s, ok := fp.grpc(consulInfo{ + "DebugConfig": {"GRPCPort": 8502.0}, // JSON numbers are floats + }) + require.True(t, ok) + require.Equal(t, "8502", s) + }) + + t.Run("grpc disabled", func(t *testing.T) { + s, ok := fp.grpc(consulInfo{ + "DebugConfig": {"GRPCPort": -1.0}, // JSON numbers are floats + }) + require.True(t, ok) + require.Equal(t, "-1", s) + }) + + t.Run("grpc missing", func(t *testing.T) { + _, ok := fp.grpc(consulInfo{ + "DebugConfig": {}, + }) + require.False(t, ok) + }) + +} + +func TestConsulFingerprint_namespaces(t *testing.T) { + t.Parallel() + + fp := newConsulFingerPrint(t) + + t.Run("supports namespaces", func(t *testing.T) { + s, ok := fp.namespaces(consulInfo{ + "Stats": {"license": map[string]interface{}{"features": "Automated Backups, Automated Upgrades, Enhanced Read Scalability, Network Segments, Redundancy Zone, Advanced Network Federation, Namespaces, SSO, Audit Logging"}}, + }) + require.True(t, ok) + require.Equal(t, "true", s) + }) + + t.Run("no namespaces", func(t *testing.T) { + _, ok := fp.namespaces(consulInfo{ + "Stats": {"license": map[string]interface{}{"features": "Automated Backups, Automated Upgrades, Enhanced Read Scalability, Network Segments, Redundancy Zone, Advanced Network Federation, SSO, Audit Logging"}}, + }) + require.False(t, ok) + }) + + t.Run("stats missing", func(t *testing.T) { + _, ok := fp.namespaces(consulInfo{}) + require.False(t, ok) + }) + + t.Run("license missing", func(t *testing.T) { + _, ok := fp.namespaces(consulInfo{"Stats": {}}) + require.False(t, ok) + }) + + t.Run("features missing", func(t *testing.T) { + _, ok := fp.namespaces(consulInfo{"Stats": {"license": map[string]interface{}{}}}) + require.False(t, ok) + }) +} + +func TestConsulFingerprint_Fingerprint_oss(t *testing.T) { cf := newConsulFingerPrint(t) - ts, cfg := fakeConsul(fakeConsulPayload(t, "test_fixtures/consul/agent_self.json")) + ts, cfg := fakeConsul(fakeConsulPayload(t, "test_fixtures/consul/agent_self_oss.json")) defer ts.Close() node := &structs.Node{Attributes: make(map[string]string)} @@ -282,6 +377,8 @@ func TestConsulFingerprint_Fingerprint(t *testing.T) { "consul.server": "true", "consul.sku": "oss", "consul.version": "1.9.5", + "consul.connect": "true", + "consul.grpc": "8502", "unique.consul.name": "HAL9000", }, resp.Attributes) require.True(t, resp.Detected) @@ -298,19 +395,24 @@ func TestConsulFingerprint_Fingerprint(t *testing.T) { node.Attributes["consul.server"] = "foo" node.Attributes["consul.sku"] = "foo" node.Attributes["consul.version"] = "foo" + node.Attributes["consul.connect"] = "foo" + node.Attributes["connect.grpc"] = "foo" node.Attributes["unique.consul.name"] = "foo" // execute second query with error err2 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp2) require.NoError(t, err2) // does not return error require.Equal(t, map[string]string{ // attributes set empty - "consul.datacenter": "", - "consul.revision": "", - "consul.segment": "", - "consul.server": "", - "consul.sku": "", - "consul.version": "", - "unique.consul.name": "", + "consul.datacenter": "", + "consul.revision": "", + "consul.segment": "", + "consul.server": "", + "consul.sku": "", + "consul.version": "", + "unique.consul.name": "", + "consul.connect": "", + "consul.grpc": "", + "consul.ft.namespaces": "", }, resp2.Attributes) require.True(t, resp.Detected) // never downgrade @@ -328,6 +430,8 @@ func TestConsulFingerprint_Fingerprint(t *testing.T) { "consul.server": "true", "consul.sku": "oss", "consul.version": "1.9.5", + "consul.connect": "true", + "consul.grpc": "8502", "unique.consul.name": "HAL9000", }, resp3.Attributes) @@ -335,3 +439,91 @@ func TestConsulFingerprint_Fingerprint(t *testing.T) { require.Equal(t, consulAvailable, cf.lastState) require.True(t, resp.Detected) } + +func TestConsulFingerprint_Fingerprint_ent(t *testing.T) { + cf := newConsulFingerPrint(t) + + ts, cfg := fakeConsul(fakeConsulPayload(t, "test_fixtures/consul/agent_self_ent.json")) + defer ts.Close() + + node := &structs.Node{Attributes: make(map[string]string)} + + // consul not available before first run + require.Equal(t, consulUnavailable, cf.lastState) + + // execute first query with good response + var resp FingerprintResponse + err := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp) + require.NoError(t, err) + require.Equal(t, map[string]string{ + "consul.datacenter": "dc1", + "consul.revision": "22ce6c6ad", + "consul.segment": "seg1", + "consul.server": "true", + "consul.sku": "ent", + "consul.version": "1.9.5+ent", + "consul.ft.namespaces": "true", + "consul.connect": "true", + "consul.grpc": "8502", + "unique.consul.name": "HAL9000", + }, resp.Attributes) + require.True(t, resp.Detected) + + // consul now available + require.Equal(t, consulAvailable, cf.lastState) + + var resp2 FingerprintResponse + + // pretend attributes set for failing request + node.Attributes["consul.datacenter"] = "foo" + node.Attributes["consul.revision"] = "foo" + node.Attributes["consul.segment"] = "foo" + node.Attributes["consul.server"] = "foo" + node.Attributes["consul.sku"] = "foo" + node.Attributes["consul.version"] = "foo" + node.Attributes["consul.ft.namespaces"] = "foo" + node.Attributes["consul.connect"] = "foo" + node.Attributes["connect.grpc"] = "foo" + node.Attributes["unique.consul.name"] = "foo" + + // execute second query with error + err2 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp2) + require.NoError(t, err2) // does not return error + require.Equal(t, map[string]string{ // attributes set empty + "consul.datacenter": "", + "consul.revision": "", + "consul.segment": "", + "consul.server": "", + "consul.sku": "", + "consul.version": "", + "consul.ft.namespaces": "", + "consul.connect": "", + "consul.grpc": "", + "unique.consul.name": "", + }, resp2.Attributes) + require.True(t, resp.Detected) // never downgrade + + // consul no longer available + require.Equal(t, consulUnavailable, cf.lastState) + + // execute third query no error + var resp3 FingerprintResponse + err3 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp3) + require.NoError(t, err3) + require.Equal(t, map[string]string{ + "consul.datacenter": "dc1", + "consul.revision": "22ce6c6ad", + "consul.segment": "seg1", + "consul.server": "true", + "consul.sku": "ent", + "consul.version": "1.9.5+ent", + "consul.ft.namespaces": "true", + "consul.connect": "true", + "consul.grpc": "8502", + "unique.consul.name": "HAL9000", + }, resp3.Attributes) + + // consul now available again + require.Equal(t, consulAvailable, cf.lastState) + require.True(t, resp.Detected) +} diff --git a/client/fingerprint/test_fixtures/consul/agent_self_ent.json b/client/fingerprint/test_fixtures/consul/agent_self_ent.json new file mode 100644 index 00000000000..c045688cc23 --- /dev/null +++ b/client/fingerprint/test_fixtures/consul/agent_self_ent.json @@ -0,0 +1,502 @@ +{ + "Config": { + "Datacenter": "dc1", + "NodeName": "HAL9000", + "NodeID": "39588353-8c00-d756-ce83-9ee70bfa5b3a", + "Revision": "22ce6c6ad", + "Server": true, + "Version": "1.9.5+ent" + }, + "DebugConfig": { + "ACLDatacenter": "dc1", + "ACLDefaultPolicy": "allow", + "ACLDisabledTTL": "2m0s", + "ACLDownPolicy": "extend-cache", + "ACLEnableKeyListPolicy": false, + "ACLMasterToken": "hidden", + "ACLPolicyTTL": "30s", + "ACLRoleTTL": "0s", + "ACLTokenReplication": false, + "ACLTokenTTL": "30s", + "ACLTokens": { + "ACLAgentMasterToken": "hidden", + "ACLAgentToken": "hidden", + "ACLDefaultToken": "hidden", + "ACLReplicationToken": "hidden", + "DataDir": "", + "EnablePersistence": false, + "EnterpriseConfig": { + "ACLServiceProviderTokens": [] + } + }, + "ACLsEnabled": false, + "AEInterval": "1m0s", + "AdvertiseAddrLAN": "127.0.0.1", + "AdvertiseAddrWAN": "127.0.0.1", + "AdvertiseReconnectTimeout": "0s", + "AllowWriteHTTPFrom": [], + "AutoConfig": { + "Authorizer": { + "AllowReuse": false, + "AuthMethod": { + "ACLAuthMethodEnterpriseFields": { + "NamespaceRules": [] + }, + "Config": { + "BoundAudiences": null, + "BoundIssuer": "", + "ClaimMappings": null, + "ClockSkewLeeway": 0, + "ExpirationLeeway": 0, + "JWKSCACert": "", + "JWKSURL": "", + "JWTSupportedAlgs": null, + "JWTValidationPubKeys": null, + "ListClaimMappings": null, + "NotBeforeLeeway": 0, + "OIDCDiscoveryCACert": "", + "OIDCDiscoveryURL": "" + }, + "Description": "", + "DisplayName": "", + "EnterpriseMeta": { + "Namespace": "default" + }, + "MaxTokenTTL": "0s", + "Name": "Auto Config Authorizer", + "RaftIndex": { + "CreateIndex": 0, + "ModifyIndex": 0 + }, + "TokenLocality": "", + "Type": "jwt" + }, + "ClaimAssertions": [], + "Enabled": false + }, + "DNSSANs": [], + "Enabled": false, + "IPSANs": [], + "IntroToken": "hidden", + "IntroTokenFile": "", + "ServerAddresses": [] + }, + "AutoEncryptAllowTLS": false, + "AutoEncryptDNSSAN": [], + "AutoEncryptIPSAN": [], + "AutoEncryptTLS": false, + "AutopilotCleanupDeadServers": true, + "AutopilotDisableUpgradeMigration": false, + "AutopilotLastContactThreshold": "200ms", + "AutopilotMaxTrailingLogs": 250, + "AutopilotMinQuorum": 0, + "AutopilotRedundancyZoneTag": "", + "AutopilotServerStabilizationTime": "10s", + "AutopilotUpgradeVersionTag": "", + "BindAddr": "127.0.0.1", + "Bootstrap": false, + "BootstrapExpect": 0, + "CAFile": "", + "CAPath": "", + "Cache": { + "EntryFetchMaxBurst": 2, + "EntryFetchRate": 1.7976931348623157e+308, + "Logger": null + }, + "CertFile": "", + "CheckDeregisterIntervalMin": "1m0s", + "CheckOutputMaxSize": 4096, + "CheckReapInterval": "30s", + "CheckUpdateInterval": "5m0s", + "Checks": [], + "ClientAddrs": [ + "127.0.0.1" + ], + "ConfigEntryBootstrap": [], + "ConnectCAConfig": {}, + "ConnectCAProvider": "", + "ConnectEnabled": true, + "ConnectMeshGatewayWANFederationEnabled": false, + "ConnectSidecarMaxPort": 21255, + "ConnectSidecarMinPort": 21000, + "ConnectTestCALeafRootChangeSpread": "0s", + "ConsulCoordinateUpdateBatchSize": 128, + "ConsulCoordinateUpdateMaxBatches": 5, + "ConsulCoordinateUpdatePeriod": "100ms", + "ConsulRaftElectionTimeout": "52ms", + "ConsulRaftHeartbeatTimeout": "35ms", + "ConsulRaftLeaderLeaseTimeout": "20ms", + "ConsulServerHealthInterval": "10ms", + "DNSARecordLimit": 0, + "DNSAddrs": [ + "tcp://127.0.0.1:8600", + "udp://127.0.0.1:8600" + ], + "DNSAllowStale": true, + "DNSAltDomain": "", + "DNSCacheMaxAge": "0s", + "DNSDisableCompression": false, + "DNSDomain": "consul.", + "DNSEnableTruncate": false, + "DNSMaxStale": "87600h0m0s", + "DNSNodeMetaTXT": true, + "DNSNodeTTL": "0s", + "DNSOnlyPassing": false, + "DNSPort": 8600, + "DNSRecursorTimeout": "2s", + "DNSRecursors": [], + "DNSSOA": { + "Expire": 86400, + "Minttl": 0, + "Refresh": 3600, + "Retry": 600 + }, + "DNSServiceTTL": {}, + "DNSUDPAnswerLimit": 3, + "DNSUseCache": false, + "DataDir": "", + "Datacenter": "dc1", + "DefaultQueryTime": "5m0s", + "DevMode": true, + "DisableAnonymousSignature": true, + "DisableCoordinates": false, + "DisableHTTPUnprintableCharFilter": false, + "DisableHostNodeID": true, + "DisableKeyringFile": true, + "DisableRemoteExec": true, + "DisableUpdateCheck": true, + "DiscardCheckOutput": false, + "DiscoveryMaxStale": "0s", + "EnableAgentTLSForChecks": false, + "EnableCentralServiceConfig": true, + "EnableDebug": true, + "EnableLocalScriptChecks": false, + "EnableRemoteScriptChecks": false, + "EncryptKey": "hidden", + "EncryptVerifyIncoming": true, + "EncryptVerifyOutgoing": true, + "EnterpriseRuntimeConfig": { + "ACLMSPDisableBootstrap": false, + "AuditEnabled": false, + "AuditSinks": [], + "DNSPreferNamespace": false + }, + "ExposeMaxPort": 21755, + "ExposeMinPort": 21500, + "GRPCAddrs": [ + "tcp://127.0.0.1:8502" + ], + "GRPCPort": 8502, + "GossipLANGossipInterval": "100ms", + "GossipLANGossipNodes": 3, + "GossipLANProbeInterval": "100ms", + "GossipLANProbeTimeout": "100ms", + "GossipLANRetransmitMult": 4, + "GossipLANSuspicionMult": 3, + "GossipWANGossipInterval": "100ms", + "GossipWANGossipNodes": 3, + "GossipWANProbeInterval": "100ms", + "GossipWANProbeTimeout": "100ms", + "GossipWANRetransmitMult": 4, + "GossipWANSuspicionMult": 3, + "HTTPAddrs": [ + "tcp://127.0.0.1:8500" + ], + "HTTPBlockEndpoints": [], + "HTTPMaxConnsPerClient": 200, + "HTTPMaxHeaderBytes": 0, + "HTTPPort": 8500, + "HTTPResponseHeaders": {}, + "HTTPSAddrs": [], + "HTTPSHandshakeTimeout": "5s", + "HTTPSPort": -1, + "HTTPUseCache": true, + "KVMaxValueSize": 524288, + "KeyFile": "hidden", + "LeaveDrainTime": "5s", + "LeaveOnTerm": false, + "Logging": { + "EnableSyslog": false, + "LogFilePath": "", + "LogJSON": false, + "LogLevel": "DEBUG", + "LogRotateBytes": 0, + "LogRotateDuration": "0s", + "LogRotateMaxFiles": 0, + "Name": "", + "SyslogFacility": "LOCAL0" + }, + "MaxQueryTime": "10m0s", + "NodeID": "39588353-8c00-d756-ce83-9ee70bfa5b3a", + "NodeMeta": {}, + "NodeName": "x52", + "PidFile": "", + "PrimaryDatacenter": "dc1", + "PrimaryGateways": [], + "PrimaryGatewaysInterval": "30s", + "RPCAdvertiseAddr": "tcp://127.0.0.1:8300", + "RPCBindAddr": "tcp://127.0.0.1:8300", + "RPCConfig": { + "EnableStreaming": false + }, + "RPCHandshakeTimeout": "5s", + "RPCHoldTimeout": "7s", + "RPCMaxBurst": 1000, + "RPCMaxConnsPerClient": 100, + "RPCProtocol": 2, + "RPCRateLimit": -1, + "RaftProtocol": 3, + "RaftSnapshotInterval": "0s", + "RaftSnapshotThreshold": 0, + "RaftTrailingLogs": 0, + "ReadReplica": false, + "ReconnectTimeoutLAN": "0s", + "ReconnectTimeoutWAN": "0s", + "RejoinAfterLeave": false, + "RetryJoinIntervalLAN": "30s", + "RetryJoinIntervalWAN": "30s", + "RetryJoinLAN": [], + "RetryJoinMaxAttemptsLAN": 0, + "RetryJoinMaxAttemptsWAN": 0, + "RetryJoinWAN": [], + "Revision": "22ce6c6ad", + "SegmentLimit": 64, + "SegmentName": "", + "SegmentNameLimit": 64, + "Segments": [], + "SerfAdvertiseAddrLAN": "tcp://127.0.0.1:8301", + "SerfAdvertiseAddrWAN": "tcp://127.0.0.1:8302", + "SerfAllowedCIDRsLAN": [], + "SerfAllowedCIDRsWAN": [], + "SerfBindAddrLAN": "tcp://127.0.0.1:8301", + "SerfBindAddrWAN": "tcp://127.0.0.1:8302", + "SerfPortLAN": 8301, + "SerfPortWAN": 8302, + "ServerMode": true, + "ServerName": "", + "ServerPort": 8300, + "Services": [], + "SessionTTLMin": "0s", + "SkipLeaveOnInt": true, + "StartJoinAddrsLAN": [], + "StartJoinAddrsWAN": [], + "SyncCoordinateIntervalMin": "15s", + "SyncCoordinateRateTarget": 64, + "TLSCipherSuites": [], + "TLSMinVersion": "tls12", + "TLSPreferServerCipherSuites": false, + "TaggedAddresses": { + "lan": "127.0.0.1", + "lan_ipv4": "127.0.0.1", + "wan": "127.0.0.1", + "wan_ipv4": "127.0.0.1" + }, + "Telemetry": { + "AllowedPrefixes": [], + "BlockedPrefixes": [], + "CirconusAPIApp": "", + "CirconusAPIToken": "hidden", + "CirconusAPIURL": "", + "CirconusBrokerID": "", + "CirconusBrokerSelectTag": "", + "CirconusCheckDisplayName": "", + "CirconusCheckForceMetricActivation": "", + "CirconusCheckID": "", + "CirconusCheckInstanceID": "", + "CirconusCheckSearchTag": "", + "CirconusCheckTags": "", + "CirconusSubmissionInterval": "", + "CirconusSubmissionURL": "", + "Disable": false, + "DisableCompatOneNine": false, + "DisableHostname": false, + "DogstatsdAddr": "", + "DogstatsdTags": [], + "FilterDefault": true, + "MetricsPrefix": "consul", + "PrometheusOpts": { + "CounterDefinitions": [], + "Expiration": "0s", + "GaugeDefinitions": [], + "Registerer": null, + "SummaryDefinitions": [] + }, + "StatsdAddr": "", + "StatsiteAddr": "" + }, + "TranslateWANAddrs": false, + "TxnMaxReqLen": 524288, + "UIConfig": { + "ContentPath": "/ui/", + "DashboardURLTemplates": {}, + "Dir": "", + "Enabled": true, + "MetricsProvider": "", + "MetricsProviderFiles": [], + "MetricsProviderOptionsJSON": "", + "MetricsProxy": { + "AddHeaders": [], + "BaseURL": "", + "PathAllowlist": [] + } + }, + "UnixSocketGroup": "", + "UnixSocketMode": "", + "UnixSocketUser": "", + "UseStreamingBackend": false, + "VerifyIncoming": false, + "VerifyIncomingHTTPS": false, + "VerifyIncomingRPC": false, + "VerifyOutgoing": false, + "VerifyServerHostname": false, + "Version": "1.9.5+ent", + "VersionPrerelease": "", + "Watches": [] + }, + "Coord": { + "Vec": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "Error": 1.5, + "Adjustment": 0, + "Height": 0.00001 + }, + "Member": { + "Name": "x52", + "Addr": "127.0.0.1", + "Port": 8301, + "Tags": { + "acls": "0", + "build": "1.9.5+ent:22ce6c6a", + "dc": "dc1", + "ft_fs": "1", + "ft_ns": "1", + "ft_si": "1", + "id": "39588353-8c00-d756-ce83-9ee70bfa5b3a", + "port": "8300", + "raft_vsn": "3", + "role": "consul", + "segment": "seg1", + "vsn": "2", + "vsn_max": "3", + "vsn_min": "2", + "wan_join_port": "8302" + }, + "Status": 1, + "ProtocolMin": 1, + "ProtocolMax": 5, + "ProtocolCur": 2, + "DelegateMin": 2, + "DelegateMax": 5, + "DelegateCur": 4 + }, + "Stats": { + "agent": { + "check_monitors": "0", + "check_ttls": "0", + "checks": "0", + "services": "0" + }, + "build": { + "prerelease": "", + "revision": "22ce6c6a", + "version": "1.9.5+ent" + }, + "consul": { + "acl": "disabled", + "bootstrap": "false", + "known_datacenters": "1", + "leader": "true", + "leader_addr": "127.0.0.1:8300", + "server": "true" + }, + "license": { + "customer": "temporary", + "expiration_time": "2021-06-03 16:44:05.553068285 -0500 CDT", + "features": "Automated Backups, Automated Upgrades, Enhanced Read Scalability, Network Segments, Redundancy Zone, Advanced Network Federation, Namespaces, SSO, Audit Logging", + "id": "temporary", + "install_id": "*", + "issue_time": "2021-06-03 10:44:05.553068285 -0500 CDT", + "modules": "", + "package": "premium", + "product": "consul", + "start_time": "2021-06-03 10:39:05.553068285 -0500 CDT" + }, + "raft": { + "applied_index": "14", + "commit_index": "14", + "fsm_pending": "0", + "last_contact": "0", + "last_log_index": "14", + "last_log_term": "2", + "last_snapshot_index": "0", + "last_snapshot_term": "0", + "latest_configuration": "[{Suffrage:Voter ID:39588353-8c00-d756-ce83-9ee70bfa5b3a Address:127.0.0.1:8300}]", + "latest_configuration_index": "0", + "num_peers": "0", + "protocol_version": "3", + "protocol_version_max": "3", + "protocol_version_min": "0", + "snapshot_version_max": "1", + "snapshot_version_min": "0", + "state": "Leader", + "term": "2" + }, + "runtime": { + "arch": "amd64", + "cpu_count": "24", + "goroutines": "96", + "max_procs": "24", + "os": "linux", + "version": "go1.16.4" + }, + "serf_lan": { + "coordinate_resets": "0", + "encrypted": "false", + "event_queue": "1", + "event_time": "2", + "failed": "0", + "health_score": "0", + "intent_queue": "0", + "left": "0", + "member_time": "1", + "members": "1", + "query_queue": "0", + "query_time": "1" + }, + "serf_wan": { + "coordinate_resets": "0", + "encrypted": "false", + "event_queue": "0", + "event_time": "1", + "failed": "0", + "health_score": "0", + "intent_queue": "0", + "left": "0", + "member_time": "1", + "members": "1", + "query_queue": "0", + "query_time": "1" + } + }, + "Meta": { + "consul-network-segment": "" + }, + "xDS": { + "SupportedProxies": { + "envoy": [ + "1.16.2", + "1.15.3", + "1.14.6", + "1.13.7" + ] + } + } +} diff --git a/client/fingerprint/test_fixtures/consul/agent_self.json b/client/fingerprint/test_fixtures/consul/agent_self_oss.json similarity index 100% rename from client/fingerprint/test_fixtures/consul/agent_self.json rename to client/fingerprint/test_fixtures/consul/agent_self_oss.json From c422aecce6604b843ae1d1a030359da83649e9be Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 3 Jun 2021 12:53:38 -0500 Subject: [PATCH 3/4] docs: update cl --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf5f932fc04..b0eb7428630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ IMPROVEMENTS: * cli: Added success confirmation message for `nomad volume delete` and `nomad volume deregister`. [[GH-10591](https://github.com/hashicorp/nomad/issues/10591)] * cli: Cross-namespace `nomad job` commands will now select exact matches if the selection is unambiguous. [[GH-10648](https://github.com/hashicorp/nomad/issues/10648)] +* client/fingerprint: Consul fingerprinter propes for additional enterprise and connect related attributes [[GH-10699](https://github.com/hashicorp/nomad/pull/10699)] BUG FIXES: * api: Fixed event stream connection initialization when there are no events to send [[GH-10637](https://github.com/hashicorp/nomad/issues/10637)] From b35fde468b88a7d09c31fe8ab08614e1dfd6b703 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 3 Jun 2021 15:12:23 -0500 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Tim Gross --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0eb7428630..da9b03a7987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ IMPROVEMENTS: * cli: Added success confirmation message for `nomad volume delete` and `nomad volume deregister`. [[GH-10591](https://github.com/hashicorp/nomad/issues/10591)] * cli: Cross-namespace `nomad job` commands will now select exact matches if the selection is unambiguous. [[GH-10648](https://github.com/hashicorp/nomad/issues/10648)] -* client/fingerprint: Consul fingerprinter propes for additional enterprise and connect related attributes [[GH-10699](https://github.com/hashicorp/nomad/pull/10699)] +* client/fingerprint: Consul fingerprinter probes for additional enterprise and connect related attributes [[GH-10699](https://github.com/hashicorp/nomad/pull/10699)] BUG FIXES: * api: Fixed event stream connection initialization when there are no events to send [[GH-10637](https://github.com/hashicorp/nomad/issues/10637)]