Skip to content

Commit

Permalink
feat: user-defined GameServers' lifecycle
Browse files Browse the repository at this point in the history
Signed-off-by: ChrisLiu <[email protected]>
  • Loading branch information
chrisliu1995 committed Dec 13, 2023
1 parent 250bd86 commit e6b4829
Show file tree
Hide file tree
Showing 9 changed files with 401 additions and 161 deletions.
8 changes: 8 additions & 0 deletions apis/v1alpha1/gameserverset_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,16 @@ type GameServerTemplate struct {
// +kubebuilder:validation:Schemaless
corev1.PodTemplateSpec `json:",inline"`
VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
Owner GameServerOwner `json:"owner,omitempty"`
}

type GameServerOwner string

const (
PodOwner GameServerOwner = "Pod"
GameServerSetOwner GameServerOwner = "GameServerSet"
)

type Network struct {
NetworkType string `json:"networkType,omitempty"`
NetworkConf []NetworkConfParams `json:"networkConf,omitempty"`
Expand Down
2 changes: 2 additions & 0 deletions config/crd/bases/game.kruise.io_gameserversets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ spec:
description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
Important: Run "make" to regenerate code after modifying this file'
properties:
owner:
type: string
volumeClaimTemplates:
items:
description: PersistentVolumeClaim is a user's request for and
Expand Down
79 changes: 26 additions & 53 deletions pkg/controllers/gameserver/gameserver_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -173,8 +172,15 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}

if podFound && !gsFound {
err := r.initGameServer(pod)
if err != nil && !errors.IsAlreadyExists(err) && !errors.IsNotFound(err) {
gss, err := r.getGameServerSet(pod)
if err != nil {
if errors.IsNotFound(err) {
return reconcile.Result{}, nil
}
return reconcile.Result{}, err

Check warning on line 180 in pkg/controllers/gameserver/gameserver_controller.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserver/gameserver_controller.go#L175-L180

Added lines #L175 - L180 were not covered by tests
}
err = r.initGameServerByPod(gss, pod)
if err != nil && !errors.IsAlreadyExists(err) {

Check warning on line 183 in pkg/controllers/gameserver/gameserver_controller.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserver/gameserver_controller.go#L182-L183

Added lines #L182 - L183 were not covered by tests
klog.Errorf("failed to create GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error())
return reconcile.Result{}, err
}
Expand Down Expand Up @@ -237,57 +243,24 @@ func (r *GameServerReconciler) getGameServerSet(pod *corev1.Pod) (*gamekruiseiov
return gss, err
}

func (r *GameServerReconciler) initGameServer(pod *corev1.Pod) error {
gs := &gamekruiseiov1alpha1.GameServer{}
gs.Name = pod.GetName()
gs.Namespace = pod.GetNamespace()

// set owner reference
gss, err := r.getGameServerSet(pod)
if err != nil {
return err
}
ors := make([]metav1.OwnerReference, 0)
or := metav1.OwnerReference{
APIVersion: gss.APIVersion,
Kind: gss.Kind,
Name: gss.GetName(),
UID: gss.GetUID(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
}
ors = append(ors, or)
gs.OwnerReferences = ors

// set Labels
gsLabels := gss.Spec.GameServerTemplate.GetLabels()
if gsLabels == nil {
gsLabels = make(map[string]string)
}
gsLabels[gamekruiseiov1alpha1.GameServerOwnerGssKey] = gss.GetName()
gs.SetLabels(gsLabels)

// set Annotations
gsAnnotations := gss.Spec.GameServerTemplate.GetAnnotations()
if gsAnnotations == nil {
gsAnnotations = make(map[string]string)
func (r *GameServerReconciler) initGameServerByPod(gss *gamekruiseiov1alpha1.GameServerSet, pod *corev1.Pod) error {
// default fields
gs := util.InitGameServer(gss, pod.Name)

if gss.Spec.GameServerTemplate.Owner == gamekruiseiov1alpha1.PodOwner || gss.Spec.GameServerTemplate.Owner == "" {
// rewrite ownerReferences
ors := make([]metav1.OwnerReference, 0)
or := metav1.OwnerReference{
APIVersion: pod.APIVersion,
Kind: pod.Kind,
Name: pod.GetName(),
UID: pod.GetUID(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
}
ors = append(ors, or)
gs.OwnerReferences = ors

Check warning on line 262 in pkg/controllers/gameserver/gameserver_controller.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserver/gameserver_controller.go#L246-L262

Added lines #L246 - L262 were not covered by tests
}
gsAnnotations[gamekruiseiov1alpha1.GsTemplateMetadataHashKey] = util.GetGsTemplateMetadataHash(gss)
gs.SetAnnotations(gsAnnotations)

// set NetWork
gs.Spec.NetworkDisabled = false

// set OpsState
gs.Spec.OpsState = gamekruiseiov1alpha1.None

// set UpdatePriority
updatePriority := intstr.FromInt(0)
gs.Spec.UpdatePriority = &updatePriority

// set deletionPriority
deletionPriority := intstr.FromInt(0)
gs.Spec.DeletionPriority = &deletionPriority

return r.Client.Create(context.Background(), gs)
}
15 changes: 9 additions & 6 deletions pkg/controllers/gameserverset/gameserverset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,15 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques
err = r.Get(ctx, namespacedName, asts)
if err != nil {
if errors.IsNotFound(err) {
// init GameServers when owner is GameServerSet
if gss.Spec.GameServerTemplate.Owner == gamekruiseiov1alpha1.GameServerSetOwner {
newManageIds, _ := computeToScaleGs(gss.Spec.ReserveGameServerIds, []int{}, []int{}, int(*gss.Spec.Replicas), []corev1.Pod{}, gamekruiseiov1alpha1.GeneralScaleDownStrategyType)
err := SyncGameServer(gss, r.Client, newManageIds, []int{})
if err != nil {
klog.Errorf("failed to sync GameServers %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error())
return reconcile.Result{}, err
}

Check warning on line 190 in pkg/controllers/gameserverset/gameserverset_controller.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_controller.go#L183-L190

Added lines #L183 - L190 were not covered by tests
}
err = r.initAsts(gss)
if err != nil {
klog.Errorf("failed to create advanced statefulset %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error())
Expand Down Expand Up @@ -252,12 +261,6 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques
return reconcile.Result{}, err
}

err = gsm.SyncGameServerReplicas()
if err != nil {
klog.Errorf("GameServerSet %s failed to adjust the replicas of GameServers to match that of Pods in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error())
return reconcile.Result{}, err
}

return ctrl.Result{}, nil
}

Expand Down
131 changes: 82 additions & 49 deletions pkg/controllers/gameserverset/gameserverset_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,6 @@ package gameserverset

import (
"context"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/tools/record"
"sort"
"sync"
"time"

kruiseV1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -32,9 +26,13 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sort"
"strconv"
"sync"

gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/pkg/util"
Expand All @@ -47,7 +45,7 @@ type Control interface {
IsNeedToScale() bool
IsNeedToUpdateWorkload() bool
SyncPodProbeMarker() error
SyncGameServerReplicas() error
//SyncGameServerReplicas() error
GetReplicasAfterKilling() *int32
}

Expand Down Expand Up @@ -129,7 +127,15 @@ func (manager *GameServerSetManager) GameServerScale() error {
klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas)
manager.eventRecorder.Eventf(gss, corev1.EventTypeNormal, ScaleReason, "scale from %d to %d", currentReplicas, expectedReplicas)

newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType)
newManageIds, newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType)

if gss.Spec.GameServerTemplate.Owner == gameKruiseV1alpha1.GameServerSetOwner {
err := SyncGameServer(gss, c, newManageIds, util.GetIndexListFromPodList(podList))
if err != nil {
return err
}

Check warning on line 136 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L133-L136

Added lines #L133 - L136 were not covered by tests
}

asts.Spec.ReserveOrdinals = newReserveIds
asts.Spec.Replicas = gss.Spec.Replicas
asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{
Expand Down Expand Up @@ -157,7 +163,7 @@ func (manager *GameServerSetManager) GameServerScale() error {
return nil
}

func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) []int {
func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) ([]int, []int) {
workloadManageIds := util.GetIndexListFromPodList(pods)

var toAdd []int
Expand Down Expand Up @@ -215,12 +221,13 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl
}
}

newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...)
newManageIds = util.GetSliceInANotInB(newManageIds, toDelete)

if scaleDownType == gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType {
return append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...)
return newManageIds, append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...)
}

newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...)
newManageIds = util.GetSliceInANotInB(newManageIds, toDelete)
var newReserveIds []int
if len(newManageIds) != 0 {
sort.Ints(newManageIds)
Expand All @@ -231,63 +238,89 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl
}
}

return newReserveIds
return newManageIds, newReserveIds
}

func (manager *GameServerSetManager) SyncGameServerReplicas() error {
gss := manager.gameServerSet
gsList := &gameKruiseV1alpha1.GameServerList{}
err := manager.client.List(context.Background(), gsList, &client.ListOptions{
Namespace: gss.GetNamespace(),
LabelSelector: labels.SelectorFromSet(map[string]string{
gameKruiseV1alpha1.GameServerOwnerGssKey: gss.GetName(),
})})
if err != nil {
return err
}
podIds := util.GetIndexListFromPodList(manager.podList)

gsMap := make(map[int]int)
deleteIds := make([]int, 0)
for id, gs := range gsList.Items {
gsId := util.GetIndexFromGsName(gs.Name)
if !util.IsNumInList(gsId, podIds) {
gsMap[gsId] = id
deleteIds = append(deleteIds, gsId)
}
}

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
func SyncGameServer(gss *gameKruiseV1alpha1.GameServerSet, c client.Client, newManageIds, oldManageIds []int) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

errch := make(chan error, len(deleteIds))
addIds := util.GetSliceInANotInB(newManageIds, oldManageIds)
klog.Infof("GameServerSet %s/%s will add GameServer Id: %v", gss.Namespace, gss.Name, addIds)
deleteIds := util.GetSliceInANotInB(oldManageIds, newManageIds)
klog.Infof("GameServerSet %s/%s will delete GameServer Id: %v", gss.Namespace, gss.Name, deleteIds)

errch := make(chan error, len(addIds)+len(deleteIds))
var wg sync.WaitGroup
for _, gsId := range deleteIds {
for _, gsId := range append(addIds, deleteIds...) {
wg.Add(1)
id := gsId
go func(ctx context.Context) {
defer wg.Done()
defer ctx.Done()

gs := gsList.Items[gsMap[id]].DeepCopy()
gsLabels := make(map[string]string)
gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true"
patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}}
patchBytes, err := json.Marshal(patchGs)
gs := &gameKruiseV1alpha1.GameServer{}
gsName := gss.Name + "-" + strconv.Itoa(id)
err := c.Get(ctx, types.NamespacedName{
Name: gsName,
Namespace: gss.Namespace,
}, gs)
if err != nil {
if errors.IsNotFound(err) {
if util.IsNumInList(id, addIds) {
err = c.Create(ctx, util.InitGameServer(gss, gsName))
if err != nil {
errch <- err
}

Check warning on line 274 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L273-L274

Added lines #L273 - L274 were not covered by tests
klog.Infof("GameServer %s/%s added", gss.Namespace, gsName)
}
return
}
errch <- err
return

Check warning on line 280 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L280

Added line #L280 was not covered by tests
}
err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, patchBytes))
if err != nil && !errors.IsNotFound(err) {
errch <- err

if util.IsNumInList(id, addIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] == "true" {
gsLabels := make(map[string]string)
gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "false"
patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}}
patchBytes, err := json.Marshal(patchGs)
if err != nil {
errch <- err
return
}

Check warning on line 291 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L289-L291

Added lines #L289 - L291 were not covered by tests
err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes))
if err != nil && !errors.IsNotFound(err) {
errch <- err
return
}

Check warning on line 296 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L294-L296

Added lines #L294 - L296 were not covered by tests
klog.Infof("GameServer %s/%s DeletingKey turn into false", gss.Namespace, gsName)
}

if util.IsNumInList(id, deleteIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "true" {
gsLabels := make(map[string]string)
gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true"
patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}}
patchBytes, err := json.Marshal(patchGs)
if err != nil {
errch <- err
return
}

Check warning on line 308 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L306-L308

Added lines #L306 - L308 were not covered by tests
err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes))
if err != nil && !errors.IsNotFound(err) {
errch <- err
return
}

Check warning on line 313 in pkg/controllers/gameserverset/gameserverset_manager.go

View check run for this annotation

Codecov / codecov/patch

pkg/controllers/gameserverset/gameserverset_manager.go#L311-L313

Added lines #L311 - L313 were not covered by tests
klog.Infof("GameServer %s/%s DeletingKey turn into true, who will be deleted", gss.Namespace, gsName)
}

}(ctx)
}

wg.Wait()
close(errch)
err = <-errch
err := <-errch
if err != nil {
klog.Errorf("failed to delete GameServers %s in %s because of %s.", gss.GetName(), gss.GetNamespace(), err.Error())
return err
}

Expand Down
Loading

0 comments on commit e6b4829

Please sign in to comment.