Kind and Kubernetes upgrade

charts/kubefed/Chart.yaml

@@ -2,6 +2,7 @@ apiVersion: v2
 description: KubeFed helm chart
 name: kubefed
 version: 0.0.3
+kubeVersion: ">= 1.16.0"
 dependencies:
 - name: controllermanager
   version: 0.0.3

charts/kubefed/charts/controllermanager/templates/webhook.yaml

@@ -3,7 +3,7 @@
 {{- $altName1 := printf "kubefed-admission-webhook.%s" .Release.Namespace }}
 {{- $altName2 := printf "kubefed-admission-webhook.%s.svc" .Release.Namespace }}
 {{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
 # For namespace scoped deployments, create a unique cluster-scoped resource
@@ -19,6 +19,8 @@ metadata:
   {{- end }}
 webhooks:
 - name: federatedtypeconfigs.core.kubefed.io
+  admissionReviewVersions:
+    - v1
   clientConfig:
     service:
       namespace: {{ .Release.Namespace | quote }}
@@ -39,6 +41,7 @@ webhooks:
     - federatedtypeconfigs
     - federatedtypeconfigs/status
   failurePolicy: Fail
+  sideEffects: None
 {{- if and .Values.global.scope (eq .Values.global.scope "Namespaced") }}
 # For namespace scoped deployments: filter admission webhook requests for
 # resources whose namespace matches the default namespace label applied by helm
@@ -51,6 +54,8 @@ webhooks:
       name: {{ .Release.Namespace }}
 {{ end }}
 - name: kubefedclusters.core.kubefed.io
+  admissionReviewVersions:
+    - v1
   clientConfig:
     service:
       namespace: {{ .Release.Namespace | quote }}
@@ -71,13 +76,16 @@ webhooks:
     - kubefedclusters
     - kubefedclusters/status
   failurePolicy: Fail
+  sideEffects: None
 {{- if and .Values.global.scope (eq .Values.global.scope "Namespaced") }}
 # See comment above.
   namespaceSelector:
     matchLabels:
       name: {{ .Release.Namespace }}
 {{ end }}
 - name: kubefedconfigs.core.kubefed.io
+  admissionReviewVersions:
+    - v1
   clientConfig:
     service:
       namespace: {{ .Release.Namespace | quote }}
@@ -97,6 +105,7 @@ webhooks:
     resources:
     - kubefedconfigs
   failurePolicy: Fail
+  sideEffects: None
 {{- if and .Values.global.scope (eq .Values.global.scope "Namespaced") }}
 # See comment above.
   namespaceSelector:
@@ -106,7 +115,7 @@ webhooks:
 ---
 # The same comments for ValidatingWebhookConfiguration apply here to
 # MutatingWebhookConfiguration.
-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
 {{- if and .Values.global.scope (eq .Values.global.scope "Namespaced") }}
@@ -116,6 +125,8 @@ metadata:
 {{- end }}
 webhooks:
 - name: kubefedconfigs.core.kubefed.io
+  admissionReviewVersions:
+    - v1
   clientConfig:
     service:
       namespace: {{ .Release.Namespace | quote }}
@@ -134,6 +145,7 @@ webhooks:
     resources:
     - kubefedconfigs
   failurePolicy: Fail
+  sideEffects: None
 {{- if and .Values.global.scope (eq .Values.global.scope "Namespaced") }}
   namespaceSelector:
     matchLabels:

docs/development.md

@@ -34,7 +34,7 @@ help you get started.
 ### Binaries
 
 The KubeFed deployment depends on `kubebuilder`, `etcd`, `kubectl`, and
-`kube-apiserver` >= v1.13 being installed in the path. The `kubebuilder`
+`kube-apiserver` >= v1.16 being installed in the path. The `kubebuilder`
 ([v2.3.1](https://github.com/kubernetes-sigs/kubebuilder/releases/tag/v2.3.1)
 as of this writing) release packages all of these dependencies together.
 
@@ -48,7 +48,7 @@ export PATH=$(pwd)/bin:${PATH}
 
 ### kubernetes
 
-The KubeFed deployment requires kubernetes version >= 1.13. To see a detailed list of binaries required, see the prerequisites section in the [user guide](./userguide.md#prerequisites)
+The KubeFed deployment requires kubernetes version >= 1.16. To see a detailed list of binaries required, see the prerequisites section in the [user guide](./userguide.md#prerequisites)
 
 ## Prerequisites
 

docs/environments/kind.md

@@ -43,17 +43,17 @@ script if you'd like to change the default:
 NUM_CLUSTERS=<num> ./scripts/create-clusters.sh
 ```
 
-The `KIND_TAG` is `v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b` by default.
-Image `kindest/node:v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b` is used as
+The `KIND_TAG` is `v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6` by default.
+Image `kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6` is used as
 node docker image for booting the cluster.
 
 You can use `KIND_IMAGE` or `KIND_TAG` to specify the image as you want.
 ```bash
-KIND_TAG=v1.18.8 ./scripts/create-clusters.sh
+KIND_TAG=v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b ./scripts/create-clusters.sh
 ```
 
 ```bash
-KIND_IMAGE=kindest/node:v1.18.8 ./scripts/create-clusters.sh
+KIND_IMAGE=kindest/node:v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b ./scripts/create-clusters.sh
 ```
 
 ## Delete Clusters

pkg/controller/sync/dispatch/retain.go

@@ -59,7 +59,7 @@ func retainServiceFields(desiredObj, clusterObj *unstructured.Unstructured) erro
 
 	// ClusterIP and NodePort are allocated to Service by cluster, so retain the same if any while updating
 
-	// Retain clusterip
+	// Retain clusterip and clusterips
 	clusterIP, ok, err := unstructured.NestedString(clusterObj.Object, "spec", "clusterIP")
 	if err != nil {
 		return errors.Wrap(err, "Error retrieving clusterIP from cluster service")
@@ -71,6 +71,17 @@ func retainServiceFields(desiredObj, clusterObj *unstructured.Unstructured) erro
 			return errors.Wrap(err, "Error setting clusterIP for service")
 		}
 	}
+	clusterIPs, ok, err := unstructured.NestedStringSlice(clusterObj.Object, "spec", "clusterIPs")
+	if err != nil {
+		return errors.Wrap(err, "Error retrieving clusterIPs from cluster service")
+	}
+	// !ok could indicate that cluster ips was not assigned
+	if ok && len(clusterIPs) > 0 {
+		err := unstructured.SetNestedStringSlice(desiredObj.Object, clusterIPs, "spec", "clusterIPs")
+		if err != nil {
+			return errors.Wrap(err, "Error setting clusterIPs for service")
+		}
+	}
 
 	// Retain nodeports
 	clusterPorts, ok, err := unstructured.NestedSlice(clusterObj.Object, "spec", "ports")

pkg/controller/sync/dispatch/retain_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package dispatch
 
 import (
+	"reflect"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -171,3 +172,121 @@ func TestRetainHealthCheckNodePortInServiceFields(t *testing.T) {
 		})
 	}
 }
+
+func TestRetainClusterIPsInServiceFields(t *testing.T) {
+	tests := []struct {
+		name                    string
+		desiredObj              *unstructured.Unstructured
+		clusterObj              *unstructured.Unstructured
+		retainSucceed           bool
+		expectedClusterIPValue  *string
+		expectedClusterIPsValue []string
+	}{
+		{
+			"cluster object has no clusterIP or clusterIPs",
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{},
+			},
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{},
+			},
+			true,
+			nil,
+			nil,
+		},
+		{
+			"cluster object has clusterIP",
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{},
+			},
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"spec": map[string]interface{}{
+						"clusterIP": -1000,
+					},
+				},
+			},
+			false,
+			nil,
+			nil,
+		},
+		{
+			"cluster object has clusterIP only",
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{},
+			},
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"spec": map[string]interface{}{
+						"clusterIP": "1.2.3.4",
+					},
+				},
+			},
+			true,
+			pointer.String("1.2.3.4"),
+			nil,
+		},
+		{
+			"cluster object has clusterIPs only",
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{},
+			},
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"spec": map[string]interface{}{
+						"clusterIPs": []interface{}{"1.2.3.4", "5.6.7.8"},
+					},
+				},
+			},
+			true,
+			nil,
+			[]string{"1.2.3.4", "5.6.7.8"},
+		},
+		{
+			"cluster object has both clusterIP and clusterIPs",
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{},
+			},
+			&unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"spec": map[string]interface{}{
+						"clusterIP":  "1.2.3.4",
+						"clusterIPs": []interface{}{"5.6.7.8", "9.10.11.12"},
+					},
+				},
+			},
+			true,
+			pointer.String("1.2.3.4"),
+			[]string{"5.6.7.8", "9.10.11.12"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := retainServiceFields(test.desiredObj, test.clusterObj); (err == nil) != test.retainSucceed {
+				t.Fatalf("test %s fails: unexpected returned error %v", test.name, err)
+			}
+
+			currentClusterIPValue, ok, err := unstructured.NestedString(test.desiredObj.Object, "spec", "clusterIP")
+			if err != nil {
+				t.Fatalf("test %s fails: %v", test.name, err)
+			}
+			if !ok && test.expectedClusterIPValue != nil {
+				t.Fatalf("test %s fails: expect specified clusterIP but not found", test.name)
+			}
+			if ok && (test.expectedClusterIPValue == nil || *test.expectedClusterIPValue != currentClusterIPValue) {
+				t.Fatalf("test %s fails: unexpected current clusterIP %s", test.name, currentClusterIPValue)
+			}
+
+			currentClusterIPsValue, ok, err := unstructured.NestedStringSlice(test.desiredObj.Object, "spec", "clusterIPs")
+			if err != nil {
+				t.Fatalf("test %s fails: %v", test.name, err)
+			}
+			if !ok && test.expectedClusterIPsValue != nil {
+				t.Fatalf("test %s fails: expect specified clusterIPs but not found", test.name)
+			}
+			if ok && !reflect.DeepEqual(test.expectedClusterIPsValue, currentClusterIPsValue) {
+				t.Fatalf("test %s fails: unexpected current clusterIPs %v", test.name, currentClusterIPsValue)
+			}
+		})
+	}
+}

pkg/kubefedctl/federate/federate.go

@@ -390,6 +390,7 @@ func FederatedResourceFromTargetResource(typeConfig typeconfig.Interface, resour
 				}
 			}
 			unstructured.RemoveNestedField(targetResource.Object, "spec", "clusterIP")
+			unstructured.RemoveNestedField(targetResource.Object, "spec", "clusterIPs")
 		}
 	}
 

pkg/schedulingtypes/replicascheduler.go

@@ -198,7 +198,7 @@ func (s *ReplicaScheduler) Reconcile(obj runtimeclient.Object, qualifiedName ctl
 
 		resultClusters, err := plugin.(*Plugin).GetResourceClusters(qualifiedName, fedClusters)
 		if err != nil {
-			runtime.HandleError(errors.Wrapf(err, "Failed to get prefrerred clusters while reconciling RSP named %q", key))
+			runtime.HandleError(errors.Wrapf(err, "Failed to get preferred clusters while reconciling RSP named %q", key))
 			return ctlutil.StatusError
 		}
 

scripts/create-clusters.sh

@@ -25,7 +25,7 @@ set -o pipefail
 source "${BASH_SOURCE%/*}/util.sh"
 NUM_CLUSTERS="${NUM_CLUSTERS:-2}"
 KIND_IMAGE="${KIND_IMAGE:-}"
-KIND_TAG="${KIND_TAG:-v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b}"
+KIND_TAG="${KIND_TAG:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}"
 OS="$(uname)"
 
 function create-clusters() {

scripts/download-e2e-binaries.sh

@@ -36,7 +36,7 @@ mkdir -p "${dest_dir}"
 
 # kind
 platform="$(uname -s|tr A-Z a-z)"
-kind_version="v0.9.0"
+kind_version="v0.11.1"
 kind_path="${dest_dir}/kind"
 kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${kind_version}/kind-${platform}-amd64"
 curl -fLo "${kind_path}" "${kind_url}" && chmod +x "${kind_path}"

scripts/pre-commit.sh

@@ -175,7 +175,7 @@ run-unit-tests
 echo "Downloading e2e test dependencies"
 ./scripts/download-e2e-binaries.sh
 
-KIND_TAG="v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b" ./scripts/create-clusters.sh
+KIND_TAG="v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6" ./scripts/create-clusters.sh
 
 declare -a join_cluster_list=()
 if [[ -z "${JOIN_CLUSTERS}" ]]; then

test/common/crudtester.go

@@ -630,6 +630,14 @@ func (c *FederatedTypeCrudTester) waitForResource(client util.ResourceClient, qu
 					c.tl.Fatalf("Failed to apply json patch: %v", err)
 				}
 
+				// Kubernetes 1.21 introduced a label kubernetes.io/metadata.name to all namespaces so regardless of what we
+				// override we should always add this label here to this check.
+				if expectedClusterObject.GetObjectKind().GroupVersionKind() == apiv1.SchemeGroupVersion.WithKind("Namespace") {
+					labels := expectedClusterObject.GetLabels()
+					labels[apiv1.LabelMetadataName] = expectedClusterObject.GetName()
+					expectedClusterObject.SetLabels(labels)
+				}
+
 				expectedClusterObjectJSON, err := expectedClusterObject.MarshalJSON()
 				if err != nil {
 					c.tl.Fatalf("Failed to marshal expected cluster object to json: %v", err)