Add LastModifyTime to catalog entities

agent/acl_test.go

@@ -80,7 +80,7 @@ func NewTestACLAgent(name string, hcl string, resolveFn func(string) (acl.Author
 	agent.MemSink = metrics.NewInmemSink(1*time.Second, time.Minute)
 
 	a.Agent.delegate = a
-	a.Agent.State = local.NewState(LocalConfig(a.Config), a.Agent.logger, a.Agent.tokens)
+	a.Agent.State = local.NewState(LocalConfig(a.Config), a.Agent.logger, a.Agent.tokens, time.Now)
 	a.Agent.State.TriggerSyncChanges = func() {}
 	return a
 }

agent/agent.go

@@ -249,6 +249,9 @@ type Agent struct {
 	// grpcServer is the server instance used currently to serve xDS API for
 	// Envoy.
 	grpcServer *grpc.Server
+
+	// clock is used to timestamp healthchecks
+	clock func() time.Time
 }
 
 func New(c *config.RuntimeConfig) (*Agent, error) {
@@ -277,6 +280,7 @@ func New(c *config.RuntimeConfig) (*Agent, error) {
 		shutdownCh:      make(chan struct{}),
 		endpoints:       make(map[string]string),
 		tokens:          new(token.Store),
+		clock:           time.Now,
 	}
 
 	if err := a.initializeACLs(); err != nil {
@@ -363,7 +367,7 @@ func (a *Agent) Start() error {
 	}
 
 	// create the local state
-	a.State = local.NewState(LocalConfig(c), a.logger, a.tokens)
+	a.State = local.NewState(LocalConfig(c), a.logger, a.tokens, a.clock)
 
 	// create the state synchronization manager which performs
 	// regular and on-demand state synchronizations (anti-entropy).
@@ -385,7 +389,7 @@ func (a *Agent) Start() error {
 
 	// Setup either the client or the server.
 	if c.ServerMode {
-		server, err := consul.NewServerLogger(consulCfg, a.logger, a.tokens)
+		server, err := consul.NewServerLogger(consulCfg, a.logger, a.tokens, a.clock)
 		if err != nil {
 			return fmt.Errorf("Failed to start Consul server: %v", err)
 		}
@@ -2799,6 +2803,7 @@ func (a *Agent) loadCheckState(check *structs.HealthCheck) error {
 	// Restore the fields from the state
 	check.Output = p.Output
 	check.Status = p.Status
+	check.LastStatusModifyTime = p.LastStatusModifyTime
 	return nil
 }
 

agent/agent_test.go

@@ -451,6 +451,8 @@ func TestAgent_AddService(t *testing.T) {
 			for k, v := range tt.healthChks {
 				t.Run(k, func(t *testing.T) {
 					got, want := a.State.Checks()[types.CheckID(k)], v
+					want.LastStatusModifyTime = a.Now
+					fmt.Println(want.LastStatusModifyTime, got.LastStatusModifyTime)
 					verify.Values(t, k, got, want)
 				})
 			}
@@ -653,9 +655,11 @@ func TestAgent_RemoveServiceRemovesAllChecks(t *testing.T) {
 	}
 
 	// check that both checks are there
+	hchk1.LastStatusModifyTime = a.Now
 	if got, want := a.State.Checks()["chk1"], hchk1; !verify.Values(t, "", got, want) {
 		t.FailNow()
 	}
+	hchk2.LastStatusModifyTime = a.Now
 	if got, want := a.State.Checks()["chk2"], hchk2; !verify.Values(t, "", got, want) {
 		t.FailNow()
 	}
@@ -1978,11 +1982,12 @@ func TestAgent_PurgeCheckOnDuplicate(t *testing.T) {
 		t.Fatalf("missing check registration")
 	}
 	expected := &structs.HealthCheck{
-		Node:    a2.Config.NodeName,
-		CheckID: "mem",
-		Name:    "memory check",
-		Status:  api.HealthCritical,
-		Notes:   "my cool notes",
+		Node:                 a2.Config.NodeName,
+		CheckID:              "mem",
+		Name:                 "memory check",
+		Status:               api.HealthCritical,
+		Notes:                "my cool notes",
+		LastStatusModifyTime: a2.Now,
 	}
 	if got, want := result, expected; !verify.Values(t, "", got, want) {
 		t.FailNow()

agent/check.go

@@ -1,6 +1,8 @@
 package agent
 
 import (
+	"time"
+
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/types"
 )
@@ -18,8 +20,9 @@ type persistedCheck struct {
 // expiration timestamp which is used to determine staleness on later
 // agent restarts.
 type persistedCheckState struct {
-	CheckID types.CheckID
-	Output  string
-	Status  string
-	Expires int64
+	CheckID              types.CheckID
+	Output               string
+	Status               string
+	LastStatusModifyTime time.Time
+	Expires              int64
 }

agent/consul/client_test.go

@@ -365,7 +365,7 @@ func TestClient_RPC_TLS(t *testing.T) {
 	conf1.VerifyIncoming = true
 	conf1.VerifyOutgoing = true
 	configureTLS(conf1)
-	s1, err := NewServer(conf1)
+	s1, err := NewServer(conf1, time.Now)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -408,7 +408,7 @@ func TestClient_RPC_TLS(t *testing.T) {
 func TestClient_RPC_RateLimit(t *testing.T) {
 	t.Parallel()
 	dir1, conf1 := testServerConfig(t)
-	s1, err := NewServer(conf1)
+	s1, err := NewServer(conf1, time.Now)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -518,7 +518,7 @@ func TestClient_SnapshotRPC_TLS(t *testing.T) {
 	conf1.VerifyIncoming = true
 	conf1.VerifyOutgoing = true
 	configureTLS(conf1)
-	s1, err := NewServer(conf1)
+	s1, err := NewServer(conf1, time.Now)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}

agent/consul/leader.go

@@ -1311,11 +1311,12 @@ AFTER_CHECK:
 		Address:    member.Addr.String(),
 		Service:    service,
 		Check: &structs.HealthCheck{
-			Node:    member.Name,
-			CheckID: structs.SerfCheckID,
-			Name:    structs.SerfCheckName,
-			Status:  api.HealthPassing,
-			Output:  structs.SerfCheckAliveOutput,
+			Node:                 member.Name,
+			CheckID:              structs.SerfCheckID,
+			Name:                 structs.SerfCheckName,
+			Status:               api.HealthPassing,
+			Output:               structs.SerfCheckAliveOutput,
+			LastStatusModifyTime: s.clock(),
 		},
 
 		// If there's existing information about the node, do not
@@ -1356,11 +1357,12 @@ func (s *Server) handleFailedMember(member serf.Member) error {
 		ID:         types.NodeID(member.Tags["id"]),
 		Address:    member.Addr.String(),
 		Check: &structs.HealthCheck{
-			Node:    member.Name,
-			CheckID: structs.SerfCheckID,
-			Name:    structs.SerfCheckName,
-			Status:  api.HealthCritical,
-			Output:  structs.SerfCheckFailedOutput,
+			Node:                 member.Name,
+			CheckID:              structs.SerfCheckID,
+			Name:                 structs.SerfCheckName,
+			Status:               api.HealthCritical,
+			Output:               structs.SerfCheckFailedOutput,
+			LastStatusModifyTime: s.clock(),
 		},
 
 		// If there's existing information about the node, do not

agent/consul/server.go

@@ -248,17 +248,20 @@ type Server struct {
 	shutdownCh   chan struct{}
 	shutdownLock sync.Mutex
 
+	// clock is used to timestamp healthchecks
+	clock func() time.Time
+
 	// embedded struct to hold all the enterprise specific data
 	EnterpriseServer
 }
 
-func NewServer(config *Config) (*Server, error) {
-	return NewServerLogger(config, nil, new(token.Store))
+func NewServer(config *Config, clock func() time.Time) (*Server, error) {
+	return NewServerLogger(config, nil, new(token.Store), clock)
 }
 
 // NewServer is used to construct a new Consul server from the
 // configuration, potentially returning an error
-func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store) (*Server, error) {
+func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store, clock func() time.Time) (*Server, error) {
 	// Check the protocol version.
 	if err := config.CheckProtocolVersion(); err != nil {
 		return nil, err
@@ -346,6 +349,7 @@ func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store) (*
 		tombstoneGC:      gc,
 		serverLookup:     NewServerLookup(),
 		shutdownCh:       shutdownCh,
+		clock:            clock,
 	}
 
 	// Initialize enterprise specific server functionality

agent/consul/server_test.go

@@ -175,7 +175,7 @@ func newServer(c *Config) (*Server, error) {
 		w = os.Stderr
 	}
 	logger := log.New(w, c.NodeName+" - ", log.LstdFlags|log.Lmicroseconds)
-	srv, err := NewServerLogger(c, logger, new(token.Store))
+	srv, err := NewServerLogger(c, logger, new(token.Store), time.Now)
 	if err != nil {
 		return nil, err
 	}

agent/consul/state/catalog_test.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
@@ -2130,16 +2131,18 @@ func TestStateStore_Service_Snapshot(t *testing.T) {
 func TestStateStore_EnsureCheck(t *testing.T) {
 	s := testStateStore(t)
 
+	now := time.Now()
 	// Create a check associated with the node
 	check := &structs.HealthCheck{
-		Node:        "node1",
-		CheckID:     "check1",
-		Name:        "redis check",
-		Status:      api.HealthPassing,
-		Notes:       "test check",
-		Output:      "aaa",
-		ServiceID:   "service1",
-		ServiceName: "redis",
+		Node:                 "node1",
+		CheckID:              "check1",
+		Name:                 "redis check",
+		Status:               api.HealthPassing,
+		Notes:                "test check",
+		Output:               "aaa",
+		ServiceID:            "service1",
+		ServiceName:          "redis",
+		LastStatusModifyTime: now,
 	}
 
 	// Creating a check without a node returns error

agent/local/state.go

@@ -214,10 +214,12 @@ type State struct {
 	// are sent a message each time a proxy changes via Add or RemoveProxy.
 	managedProxies       map[string]*ManagedProxy
 	managedProxyHandlers map[chan<- struct{}]struct{}
+
+	clock func() time.Time
 }
 
 // NewState creates a new local state for the agent.
-func NewState(c Config, lg *log.Logger, tokens *token.Store) *State {
+func NewState(c Config, lg *log.Logger, tokens *token.Store, clock func() time.Time) *State {
 	l := &State{
 		config:               c,
 		logger:               lg,
@@ -229,6 +231,7 @@ func NewState(c Config, lg *log.Logger, tokens *token.Store) *State {
 		notifyHandlers:       make(map[chan<- struct{}]struct{}),
 		managedProxies:       make(map[string]*ManagedProxy),
 		managedProxyHandlers: make(map[chan<- struct{}]struct{}),
+		clock:                clock,
 	}
 	l.SetDiscardCheckOutput(c.DiscardCheckOutput)
 	return l
@@ -434,6 +437,9 @@ func (l *State) AddCheck(check *structs.HealthCheck, token string) error {
 	// hard-set the node name
 	check.Node = l.config.NodeName
 
+	// init last status modify time
+	check.LastStatusModifyTime = l.clock().UTC()
+
 	l.SetCheckState(&CheckState{
 		Check: check,
 		Token: token,
@@ -572,6 +578,7 @@ func (l *State) UpdateCheck(id types.CheckID, status, output string) {
 	// Update status and mark out of sync
 	c.Check.Status = status
 	c.Check.Output = output
+	c.Check.LastStatusModifyTime = l.clock().UTC()
 	c.InSync = false
 	l.TriggerSyncChanges()
 }