Merge pull request #2034 from sawsa307/neg-sync-metric

Neg sync metric

pkg/neg/metrics/neg_metrics_collector.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/util/wait"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/klog/v2"
@@ -69,6 +70,8 @@ func FakeSyncerMetrics() *SyncerMetrics {
 
 // RegisterSyncerMetrics registers syncer related metrics
 func RegisterSyncerMetrics() {
+	prometheus.MustRegister(syncerSyncResult)
+	prometheus.MustRegister(syncerSyncerState)
 }
 
 func (sm *SyncerMetrics) Run(stopCh <-chan struct{}) {
@@ -86,7 +89,11 @@ func (sm *SyncerMetrics) export() {
 	lpMetrics := sm.computeLabelMetrics()
 	NumberOfEndpoints.WithLabelValues(totalEndpoints).Set(float64(lpMetrics.NumberOfEndpoints))
 	NumberOfEndpoints.WithLabelValues(epWithAnnotation).Set(float64(lpMetrics.EndpointsWithAnnotation))
-	sm.logger.V(3).Info("Exporting syncer related metrics", "Number of Endpoints", lpMetrics.NumberOfEndpoints)
+
+	stateCount, syncerCount := sm.computeSyncerStateMetrics()
+	PublishSyncerStateMetrics(stateCount)
+
+	sm.logger.V(3).Info("Exporting syncer related metrics", "Syncer count", syncerCount, "Number of Endpoints", lpMetrics.NumberOfEndpoints)
 }
 
 // UpdateSyncerStatusInMetrics update the status of syncer based on the error
@@ -96,6 +103,7 @@ func (sm *SyncerMetrics) UpdateSyncerStatusInMetrics(key negtypes.NegSyncerKey,
 		syncErr := negtypes.ClassifyError(err)
 		reason = syncErr.Reason
 	}
+	syncerSyncResult.WithLabelValues(string(reason)).Inc()
 	sm.mu.Lock()
 	defer sm.mu.Unlock()
 	if sm.syncerStatusMap == nil {
@@ -144,3 +152,16 @@ func (sm *SyncerMetrics) computeLabelMetrics() LabelPropagationMetrics {
 	}
 	return lpMetrics
 }
+
+func (sm *SyncerMetrics) computeSyncerStateMetrics() (*syncerStateCount, int) {
+	sm.mu.Lock()
+	defer sm.mu.Unlock()
+
+	stateCount := &syncerStateCount{}
+	syncerCount := 0
+	for _, syncerState := range sm.syncerStatusMap {
+		stateCount.inc(syncerState)
+		syncerCount++
+	}
+	return stateCount, syncerCount
+}

pkg/neg/metrics/syncer_metrics.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	negtypes "k8s.io/ingress-gce/pkg/neg/types"
+)
+
+const (
+	syncResultLabel = "result"
+	syncResultKey   = "sync_result"
+
+	syncerStateLabel = "state"
+	syncerStateKey   = "syncer_state"
+
+	EPCountsDiffer           = "EndpointCountsDiffer"
+	EPNodeMissing            = "EndpointNodeMissing"
+	EPNodeNotFound           = "EndpointNodeNotFound"
+	EPPodMissing             = "EndpointPodMissing"
+	EPPodNotFound            = "EndpointPodNotFound"
+	EPPodTypeAssertionFailed = "EndpointPodTypeAssertionFailed"
+	EPZoneMissing            = "EndpointZoneMissing"
+	EPSEndpointCountZero     = "EndpointSliceEndpointCountZero"
+	EPCalculationCountZero   = "EndpointCalculationCountZero"
+	InvalidAPIResponse       = "InvalidAPIResponse"
+	InvalidEPAttach          = "InvalidEndpointAttach"
+	InvalidEPDetach          = "InvalidEndpointDetach"
+	NegNotFound              = "NetworkEndpointGroupNotFound"
+	CurrentNegEPNotFound     = "CurrentNEGEndpointNotFound"
+	EPSNotFound              = "EndpointSliceNotFound"
+	OtherError               = "OtherError"
+	Success                  = "Success"
+)
+
+var (
+	// syncerSyncResult tracks the count for each sync result
+	syncerSyncResult = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: negControllerSubsystem,
+			Name:      syncResultKey,
+			Help:      "Current count for each sync result",
+		},
+		[]string{syncResultLabel},
+	)
+
+	// syncerSyncerState tracks the count of syncer in different states
+	syncerSyncerState = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Subsystem: negControllerSubsystem,
+			Name:      syncerStateKey,
+			Help:      "Current count of syncers in each state",
+		},
+		[]string{syncerStateLabel},
+	)
+)
+
+type syncerStateCount struct {
+	epCountsDiffer           int
+	epNodeMissing            int
+	epNodeNotFound           int
+	epPodMissing             int
+	epPodNotFound            int
+	epPodTypeAssertionFailed int
+	epZoneMissing            int
+	epsEndpointCountZero     int
+	epCalculationCountZero   int
+	invalidAPIResponse       int
+	invalidEPAttach          int
+	invalidEPDetach          int
+	negNotFound              int
+	currentNegEPNotFound     int
+	epsNotFound              int
+	otherError               int
+	success                  int
+}
+
+func (sc *syncerStateCount) inc(reason negtypes.Reason) {
+	switch reason {
+	case negtypes.ReasonEPCountsDiffer:
+		sc.epCountsDiffer++
+	case negtypes.ReasonEPNodeMissing:
+		sc.epNodeMissing++
+	case negtypes.ReasonEPNodeNotFound:
+		sc.epNodeNotFound++
+	case negtypes.ReasonEPPodMissing:
+		sc.epPodMissing++
+	case negtypes.ReasonEPPodNotFound:
+		sc.epPodNotFound++
+	case negtypes.ReasonEPPodTypeAssertionFailed:
+		sc.epPodTypeAssertionFailed++
+	case negtypes.ReasonEPZoneMissing:
+		sc.epZoneMissing++
+	case negtypes.ReasonEPSEndpointCountZero:
+		sc.epsEndpointCountZero++
+	case negtypes.ReasonInvalidAPIResponse:
+		sc.invalidAPIResponse++
+	case negtypes.ReasonInvalidEPAttach:
+		sc.invalidEPAttach++
+	case negtypes.ReasonInvalidEPDetach:
+		sc.invalidEPDetach++
+	case negtypes.ReasonNegNotFound:
+		sc.negNotFound++
+	case negtypes.ReasonCurrentNegEPNotFound:
+		sc.currentNegEPNotFound++
+	case negtypes.ReasonEPSNotFound:
+		sc.epsNotFound++
+	case negtypes.ReasonOtherError:
+		sc.otherError++
+	case negtypes.ReasonSuccess:
+		sc.success++
+	}
+}
+
+func PublishSyncerStateMetrics(stateCount *syncerStateCount) {
+	syncerSyncerState.WithLabelValues(EPCountsDiffer).Set(float64(stateCount.epCountsDiffer))
+	syncerSyncerState.WithLabelValues(EPNodeMissing).Set(float64(stateCount.epNodeMissing))
+	syncerSyncerState.WithLabelValues(EPNodeNotFound).Set(float64(stateCount.epNodeNotFound))
+	syncerSyncerState.WithLabelValues(EPPodMissing).Set(float64(stateCount.epPodMissing))
+	syncerSyncerState.WithLabelValues(EPPodNotFound).Set(float64(stateCount.epPodNotFound))
+	syncerSyncerState.WithLabelValues(EPPodTypeAssertionFailed).Set(float64(stateCount.epPodTypeAssertionFailed))
+	syncerSyncerState.WithLabelValues(EPZoneMissing).Set(float64(stateCount.epZoneMissing))
+	syncerSyncerState.WithLabelValues(EPSEndpointCountZero).Set(float64(stateCount.epsEndpointCountZero))
+	syncerSyncerState.WithLabelValues(EPCalculationCountZero).Set(float64(stateCount.epCalculationCountZero))
+	syncerSyncerState.WithLabelValues(InvalidAPIResponse).Set(float64(stateCount.invalidAPIResponse))
+	syncerSyncerState.WithLabelValues(InvalidEPAttach).Set(float64(stateCount.invalidEPAttach))
+	syncerSyncerState.WithLabelValues(InvalidEPDetach).Set(float64(stateCount.invalidEPDetach))
+	syncerSyncerState.WithLabelValues(NegNotFound).Set(float64(stateCount.negNotFound))
+	syncerSyncerState.WithLabelValues(CurrentNegEPNotFound).Set(float64(stateCount.currentNegEPNotFound))
+	syncerSyncerState.WithLabelValues(EPSNotFound).Set(float64(stateCount.epsNotFound))
+	syncerSyncerState.WithLabelValues(OtherError).Set(float64(stateCount.otherError))
+	syncerSyncerState.WithLabelValues(Success).Set(float64(stateCount.success))
+}