Add controller performance metrics

.gitignore

@@ -23,5 +23,8 @@ installfilesdev
 
 .uptodate
 
+# vscode settings
+.vscode
+
 # allocator compiled plugin
 kubectl-gameserver

cmd/nodeagent/nodeagentmanager.go

@@ -452,8 +452,12 @@ func (n *NodeAgentManager) updateHealthAndStateIfNeeded(ctx context.Context, hb
 			now := metav1.Time{Time: n.nowFunc()}
 			if hb.CurrentGameState == GameStateInitializing {
 				status.ReachedInitializingOn = &now
+				timeDif := time.Now().UnixMilli() - gsd.CreationTime
+				GameServerReachedInitializingDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif))
 			} else if hb.CurrentGameState == GameStateStandingBy {
 				status.ReachedStandingByOn = &now
+				timeDif := time.Now().UnixMilli() - gsd.CreationTime
+				GameServerReachedStandingByDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif))
 			}
 		}
 

cmd/nodeagent/types.go

@@ -58,6 +58,24 @@ var (
 		Name:      "connected_players",
 		Help:      "Number of connected players per GameServer",
 	}, []string{"namespace", "ServerName", "BuildName"})
+
+	GameServerReachedStandingByDuration = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "thundernetes",
+			Name:      "gameserver_standing_by_duration",
+			Help:      "Time taken for a GameServer to reach StandingBy",
+		},
+		[]string{"BuildName"},
+	)
+
+	GameServerReachedInitializingDuration = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "thundernetes",
+			Name:      "gameserver_initialization_duration",
+			Help:      "Time taken for a GameServer to reach initialization",
+		},
+		[]string{"BuildName"},
+	)
 )
 
 // HeartbeatRequest contains data for the heartbeat request coming from the GSDK running alongside GameServer

docs/howtos/monitoring.md

@@ -84,6 +84,8 @@ There is a custom Grafana dashboard example that visualizes some of this data in
 | --- | --- | --- |
 | gameserver_states | Gauge | nodeagent |
 | connected_players | Gauge | nodeagent |
+| gameserver_initialization_duration | Gauge | nodeagent |
+| gameserver_standing_by_duration | Gauge | nodeagent |
 | gameservers_current_state_per_build | Gauge | controller-manager |
 | gameservers_created_total | Counter | controller-manager |
 | gameservers_sessionended_total | Counter | controller-manager |

pkg/operator/controllers/gameserverbuild_controller.go

@@ -183,19 +183,18 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	// calculate the total amount of servers not in the active state
 	nonActiveGameServersCount := standingByCount + initializingCount + pendingCount
 
+	// Evaluate desired number of servers against actual
+	var totalNumberOfGameServersToDelete int = 0
 	// user has decreased standingBy numbers
 	if nonActiveGameServersCount > gsb.Spec.StandingBy {
-		totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete))
-		err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
+		totalNumberOfGameServersToDelete += int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete))
 	}
-
-	// we need to check if we are above the max
+	// we also need to check if we are above the max
 	// this can happen if the user modifies the spec.Max during the GameServerBuild's lifetime
 	if nonActiveGameServersCount+activeCount > gsb.Spec.Max {
-		totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount+activeCount-gsb.Spec.Max), maxNumberOfGameServersToDelete))
+		totalNumberOfGameServersToDelete += int(math.Min(float64(totalNumberOfGameServersToDelete+(nonActiveGameServersCount+activeCount-gsb.Spec.Max)), maxNumberOfGameServersToDelete))
+	}
+	if totalNumberOfGameServersToDelete > 0 {
 		err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete)
 		if err != nil {
 			return ctrl.Result{}, err