Add controller performance metrics (#391)

* Added GameServer control performance metrics; Updated Grafana dashboard * Update monitoring documentation; Refactor non-active handling * Update yaml * Fix capitalization * Revert extra changes triggering installfile alret * Handle dereferencing; Add util funciton; Remove repeated lines * Add pointers * Decreasing time diff * PR Updates; Rename deleteSum; Fix GS state update; * Add patching exception * Change metric emission to nodeagent * Update dashboard * Revert test * Cleanup deletes * Minor tweaks * Conditional * PR Suggested changes * Remove spacing added to gameserverbuild * Remove empty line in nodeagent * Renaming * Update dashboard * Remove metric Co-authored-by: Dimitris-Ilias Gkanatsios <[email protected]>
PlayFab · Sep 26, 2022 · 15336f7 · 15336f7
1 parent e038492
commit 15336f7
Show file tree

Hide file tree

Showing 6 changed files with 1,006 additions and 695 deletions.
diff --git a/.gitignore b/.gitignore
@@ -23,5 +23,8 @@ installfilesdev
 
 .uptodate
 
+# vscode settings
+.vscode
+
 # allocator compiled plugin
 kubectl-gameserver
diff --git a/cmd/nodeagent/nodeagentmanager.go b/cmd/nodeagent/nodeagentmanager.go
@@ -452,8 +452,12 @@ func (n *NodeAgentManager) updateHealthAndStateIfNeeded(ctx context.Context, hb
 			now := metav1.Time{Time: n.nowFunc()}
 			if hb.CurrentGameState == GameStateInitializing {
 				status.ReachedInitializingOn = &now
+				timeDif := time.Now().UnixMilli() - gsd.CreationTime
+				GameServerReachedInitializingDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif))
 			} else if hb.CurrentGameState == GameStateStandingBy {
 				status.ReachedStandingByOn = &now
+				timeDif := time.Now().UnixMilli() - gsd.CreationTime
+				GameServerReachedStandingByDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif))
 			}
 		}
 

diff --git a/cmd/nodeagent/types.go b/cmd/nodeagent/types.go
@@ -58,6 +58,24 @@ var (
 		Name:      "connected_players",
 		Help:      "Number of connected players per GameServer",
 	}, []string{"namespace", "ServerName", "BuildName"})
+
+	GameServerReachedStandingByDuration = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "thundernetes",
+			Name:      "gameserver_standing_by_duration",
+			Help:      "Time taken for a GameServer to reach StandingBy",
+		},
+		[]string{"BuildName"},
+	)
+
+	GameServerReachedInitializingDuration = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "thundernetes",
+			Name:      "gameserver_initialization_duration",
+			Help:      "Time taken for a GameServer to reach initialization",
+		},
+		[]string{"BuildName"},
+	)
 )
 
 // HeartbeatRequest contains data for the heartbeat request coming from the GSDK running alongside GameServer

diff --git a/docs/howtos/monitoring.md b/docs/howtos/monitoring.md
@@ -84,6 +84,8 @@ There is a custom Grafana dashboard example that visualizes some of this data in
 | --- | --- | --- |
 | gameserver_states | Gauge | nodeagent |
 | connected_players | Gauge | nodeagent |
+| gameserver_initialization_duration | Gauge | nodeagent |
+| gameserver_standing_by_duration | Gauge | nodeagent |
 | gameservers_current_state_per_build | Gauge | controller-manager |
 | gameservers_created_total | Counter | controller-manager |
 | gameservers_sessionended_total | Counter | controller-manager |

diff --git a/pkg/operator/controllers/gameserverbuild_controller.go b/pkg/operator/controllers/gameserverbuild_controller.go
@@ -183,19 +183,18 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	// calculate the total amount of servers not in the active state
 	nonActiveGameServersCount := standingByCount + initializingCount + pendingCount
 
+	// Evaluate desired number of servers against actual
+	var totalNumberOfGameServersToDelete int = 0
 	// user has decreased standingBy numbers
 	if nonActiveGameServersCount > gsb.Spec.StandingBy {
-		totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete))
-		err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
+		totalNumberOfGameServersToDelete += int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete))
 	}
-
-	// we need to check if we are above the max
+	// we also need to check if we are above the max
 	// this can happen if the user modifies the spec.Max during the GameServerBuild's lifetime
 	if nonActiveGameServersCount+activeCount > gsb.Spec.Max {
-		totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount+activeCount-gsb.Spec.Max), maxNumberOfGameServersToDelete))
+		totalNumberOfGameServersToDelete += int(math.Min(float64(totalNumberOfGameServersToDelete+(nonActiveGameServersCount+activeCount-gsb.Spec.Max)), maxNumberOfGameServersToDelete))
+	}
+	if totalNumberOfGameServersToDelete > 0 {
 		err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete)
 		if err != nil {
 			return ctrl.Result{}, err