Add pending health status (#672)

* Add pending health status for Agent v2
nginx · May 28, 2024 · dfe7e69 · dfe7e69
1 parent 3942793
commit dfe7e69
Show file tree

Hide file tree

Showing 3 changed files with 171 additions and 23 deletions.
diff --git a/src/plugins/agent_api.go b/src/plugins/agent_api.go
@@ -75,6 +75,7 @@ type RootHandler struct {
 	isGrpcRegistered     bool
 	lastCommandSent      time.Time
 	lastMetricReportSent time.Time
+	startTime            time.Time
 }
 
 type NginxHandler struct {
@@ -252,9 +253,9 @@ func (a *AgentAPI) Subscriptions() []string {
 
 func (a *AgentAPI) createHttpServer() {
 	a.rootHandler = &RootHandler{
-		config:               a.config,
-		isGrpcRegistered:     false,
-		lastMetricReportSent: time.Now(),
+		config:           a.config,
+		isGrpcRegistered: false,
+		startTime:        time.Now(),
 	}
 
 	a.nginxHandler = &NginxHandler{
@@ -686,17 +687,19 @@ func (rh *RootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func (rh *RootHandler) healthCheck(w http.ResponseWriter) error {
 	w.WriteHeader(http.StatusOK)
 
-	overallStatus := okStatus
+	overallStatus := pendingStatus
 	checks := []HealthStatusCheck{}
 
-	registrationStatus := okStatus
-	commandServiceStatus := okStatus
-	metricsServiceStatus := okStatus
+	registrationStatus := pendingStatus
+	commandServiceStatus := pendingStatus
+	metricsServiceStatus := pendingStatus
 
 	if rh.config.IsGrpcServerConfigured() {
 		if !rh.isGrpcRegistered {
 			registrationStatus = errorStatus
 			overallStatus = errorStatus
+		} else {
+			registrationStatus = okStatus
 		}
 
 		checks = append(checks, HealthStatusCheck{
@@ -705,10 +708,18 @@ func (rh *RootHandler) healthCheck(w http.ResponseWriter) error {
 		})
 
 		timeNow := time.Now()
+		startTimeDiff := timeNow.Sub(rh.startTime)
 
-		lastCommandSentDiff := timeNow.Sub(rh.lastCommandSent)
+		if !rh.lastCommandSent.IsZero() {
+			lastCommandSentDiff := timeNow.Sub(rh.lastCommandSent)
 
-		if lastCommandSentDiff > (2 * rh.config.Dataplane.Status.PollInterval) {
+			if lastCommandSentDiff > (2 * rh.config.Dataplane.Status.PollInterval) {
+				commandServiceStatus = errorStatus
+				overallStatus = errorStatus
+			} else {
+				commandServiceStatus = okStatus
+			}
+		} else if startTimeDiff > (2 * rh.config.Dataplane.Status.PollInterval) {
 			commandServiceStatus = errorStatus
 			overallStatus = errorStatus
 		}
@@ -719,9 +730,16 @@ func (rh *RootHandler) healthCheck(w http.ResponseWriter) error {
 		})
 
 		if rh.config.IsFeatureEnabled(agent_config.FeatureMetrics) || rh.config.IsFeatureEnabled(agent_config.FeatureMetricsSender) {
-			lastMetricReportSentDiff := timeNow.Sub(rh.lastMetricReportSent)
+			if !rh.lastMetricReportSent.IsZero() {
+				lastMetricReportSentDiff := timeNow.Sub(rh.lastMetricReportSent)
 
-			if lastMetricReportSentDiff > (2 * rh.config.AgentMetrics.ReportInterval) {
+				if lastMetricReportSentDiff > (2 * rh.config.AgentMetrics.ReportInterval) {
+					metricsServiceStatus = errorStatus
+					overallStatus = errorStatus
+				} else {
+					metricsServiceStatus = okStatus
+				}
+			} else if startTimeDiff > (2 * rh.config.AgentMetrics.ReportInterval) {
 				metricsServiceStatus = errorStatus
 				overallStatus = errorStatus
 			}
@@ -733,6 +751,10 @@ func (rh *RootHandler) healthCheck(w http.ResponseWriter) error {
 		}
 	}
 
+	if registrationStatus == okStatus && commandServiceStatus == okStatus && metricsServiceStatus == okStatus {
+		overallStatus = okStatus
+	}
+
 	healthResponse := &HealthResponse{
 		Status: overallStatus,
 		Checks: checks,

diff --git a/src/plugins/agent_api_test.go b/src/plugins/agent_api_test.go
@@ -617,7 +617,59 @@ func TestRootHandler_healthCheck(t *testing.T) {
 			},
 		},
 		{
-			name: "Test 4: Metrics service connection failed",
+			name: "Test 4: Command service connection pending",
+			rootHandler: &RootHandler{
+				config:               agentConfig,
+				isGrpcRegistered:     true,
+				lastMetricReportSent: time.Now(),
+				startTime:            time.Now(),
+			},
+			expected: &HealthResponse{
+				Status: pendingStatus,
+				Checks: []HealthStatusCheck{
+					{
+						Name:   registration,
+						Status: okStatus,
+					},
+					{
+						Name:   commandConnection,
+						Status: pendingStatus,
+					},
+					{
+						Name:   metricsConnection,
+						Status: okStatus,
+					},
+				},
+			},
+		},
+		{
+			name: "Test 5: Command service never connected",
+			rootHandler: &RootHandler{
+				config:               agentConfig,
+				isGrpcRegistered:     true,
+				lastMetricReportSent: time.Now(),
+				startTime:            time.Now().AddDate(0, 0, -1),
+			},
+			expected: &HealthResponse{
+				Status: errorStatus,
+				Checks: []HealthStatusCheck{
+					{
+						Name:   registration,
+						Status: okStatus,
+					},
+					{
+						Name:   commandConnection,
+						Status: errorStatus,
+					},
+					{
+						Name:   metricsConnection,
+						Status: okStatus,
+					},
+				},
+			},
+		},
+		{
+			name: "Test 6: Metrics service connection failed",
 			rootHandler: &RootHandler{
 				config:               agentConfig,
 				isGrpcRegistered:     true,
@@ -642,6 +694,58 @@ func TestRootHandler_healthCheck(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "Test 7: Metrics service connection pending",
+			rootHandler: &RootHandler{
+				config:           agentConfig,
+				isGrpcRegistered: true,
+				lastCommandSent:  time.Now(),
+				startTime:        time.Now(),
+			},
+			expected: &HealthResponse{
+				Status: pendingStatus,
+				Checks: []HealthStatusCheck{
+					{
+						Name:   registration,
+						Status: okStatus,
+					},
+					{
+						Name:   commandConnection,
+						Status: okStatus,
+					},
+					{
+						Name:   metricsConnection,
+						Status: pendingStatus,
+					},
+				},
+			},
+		},
+		{
+			name: "Test 8: Metrics service never connected",
+			rootHandler: &RootHandler{
+				config:           agentConfig,
+				isGrpcRegistered: true,
+				lastCommandSent:  time.Now(),
+				startTime:        time.Now().AddDate(0, 0, -1),
+			},
+			expected: &HealthResponse{
+				Status: errorStatus,
+				Checks: []HealthStatusCheck{
+					{
+						Name:   registration,
+						Status: okStatus,
+					},
+					{
+						Name:   commandConnection,
+						Status: okStatus,
+					},
+					{
+						Name:   metricsConnection,
+						Status: errorStatus,
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/agent_api.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/agent_api.go