Merge pull request #43 from cmurphy/fix-labels

Fix label updates
rancher · May 4, 2021 · a7625e3 · a7625e3
2 parents d57ee5f + 1838715
commit a7625e3
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 19 deletions.
diff --git a/controller/gke-cluster-config-handler.go b/controller/gke-cluster-config-handler.go
@@ -12,6 +12,7 @@ import (
 	wranglerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
+	"k8s.io/client-go/util/retry"
 
 	gkeapi "google.golang.org/api/container/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -283,9 +284,17 @@ func (h *Handler) enqueueUpdate(config *gkev1.GKEClusterConfig) (*gkev1.GKEClust
 		h.gkeEnqueue(config.Namespace, config.Name)
 		return config, nil
 	}
-	config = config.DeepCopy()
-	config.Status.Phase = gkeConfigUpdatingPhase
-	return h.gkeCC.UpdateStatus(config)
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		var err error
+		config, err = h.gkeCC.Get(config.Namespace, config.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		config.Status.Phase = gkeConfigUpdatingPhase
+		config, err = h.gkeCC.UpdateStatus(config)
+		return err
+	})
+	return config, err
 }
 
 func (h *Handler) updateUpstreamClusterState(config *gkev1.GKEClusterConfig, upstreamSpec *gkev1.GKEClusterConfigSpec) (*gkev1.GKEClusterConfig, error) {
@@ -551,7 +560,11 @@ func BuildUpstreamClusterState(cluster *gkeapi.Cluster) (*gkev1.GKEClusterConfig
 			Enabled: false,
 		},
 		Locations: cluster.Locations,
-		Labels:    cluster.ResourceLabels,
+	}
+	if cluster.ResourceLabels == nil {
+		newSpec.Labels = make(map[string]string, 0)
+	} else {
+		newSpec.Labels = cluster.ResourceLabels
 	}
 
 	networkPolicyEnabled := false

diff --git a/internal/gke/update.go b/internal/gke/update.go
@@ -7,11 +7,13 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/rancher/gke-operator/internal/utils"
 	gkev1 "github.com/rancher/gke-operator/pkg/apis/gke.cattle.io/v1"
 	"github.com/sirupsen/logrus"
 	gkeapi "google.golang.org/api/container/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // Network Providers
@@ -373,22 +375,37 @@ func UpdateLabels(
 	if config.Spec.Labels == nil || reflect.DeepEqual(config.Spec.Labels, upstreamSpec.Labels) || (upstreamSpec.Labels == nil && len(config.Spec.Labels) == 0) {
 		return NotChanged, nil
 	}
-	cluster, err := GetCluster(ctx, client, &config.Spec)
-	if err != nil {
-		return NotChanged, err
-	}
 	logrus.Infof("updating cluster labels for cluster [%s]", config.Name)
-	_, err = client.Projects.
-		Locations.
-		Clusters.
-		SetResourceLabels(
-			ClusterRRN(config.Spec.ProjectID, Location(config.Spec.Region, config.Spec.Zone), config.Spec.ClusterName),
-			&gkeapi.SetLabelsRequest{
-				LabelFingerprint: cluster.LabelFingerprint,
-				ResourceLabels:   config.Spec.Labels,
-			},
-		).Context(ctx).
-		Do()
+	backoff := wait.Backoff{
+		Duration: 5 * time.Second,
+		Steps:    2,
+	}
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		cluster, err := GetCluster(ctx, client, &config.Spec)
+		if err != nil {
+			return false, err
+		}
+		_, err = client.Projects.
+			Locations.
+			Clusters.
+			SetResourceLabels(
+				ClusterRRN(config.Spec.ProjectID, Location(config.Spec.Region, config.Spec.Zone), config.Spec.ClusterName),
+				&gkeapi.SetLabelsRequest{
+					LabelFingerprint: cluster.LabelFingerprint,
+					ResourceLabels:   config.Spec.Labels,
+				},
+			).Context(ctx).
+			Do()
+		if err != nil && strings.Contains(err.Error(), "Labels could not be set due to fingerprint mismatch") {
+			logrus.Debug("retrying label update")
+			return false, nil
+		}
+		if err != nil {
+			logrus.Debugf("error during label update: %v", err)
+			return false, err
+		}
+		return true, nil
+	})
 	if err != nil {
 		return NotChanged, err
 	}