Refactored code

Addon upgrade/downgrade test similar to aws#1795 Added tests for addon upgrade/downgrade Changed DEFAULT version Added addon status checks Fetch latest addon version for given K8s Cluster Update kops cluster config used in weekly tests (aws#1862) * Change to kops cluster creation scripts * Add logging for retry attempt * Switch kops cluster to use docker container runtime Co-authored-by: Jayanth Varavani <[email protected]>
cgchinmay · Feb 15, 2022 · 88e166b · 88e166b
1 parent 0007251
commit 88e166b
Show file tree

Hide file tree

Showing 13 changed files with 546 additions and 197 deletions.
diff --git a/.github/workflows/weekly-cron-test.yml b/.github/workflows/weekly-cron-test.yml
@@ -2,7 +2,7 @@ name: Weekly CNI tests
 
 on:
   schedule:
-    - cron: "0 16 * * 1"  # every Monday
+    - cron: "0 16 * * 2"  # every Tuesday
 
 permissions:
   contents: read

diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
@@ -66,14 +66,18 @@ function up-test-cluster() {
 }
 
 function up-kops-cluster {
-    aws s3api create-bucket --bucket kops-cni-test-eks --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
-    curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
+    KOPS_S3_BUCKET=kops-cni-test-eks-$AWS_ACCOUNT_ID
+    echo "Using $KOPS_S3_BUCKET as kops state store"
+    aws s3api create-bucket --bucket $KOPS_S3_BUCKET --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    kops_version=$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)
+    echo "Using kops version $kops_version"
+    curl -LO https://github.com/kubernetes/kops/releases/download/$kops_version/kops-linux-amd64
     chmod +x kops-linux-amd64
     mkdir -p ~/kops_bin
     KOPS_BIN=~/kops_bin/kops
     mv kops-linux-amd64 $KOPS_BIN
     CLUSTER_NAME=kops-cni-test-cluster-${TEST_ID}.k8s.local
-    export KOPS_STATE_STORE=s3://kops-cni-test-eks
+    export KOPS_STATE_STORE=s3://${KOPS_S3_BUCKET}
 
     SSH_KEYS=~/.ssh/devopsinuse
     if [ ! -f "$SSH_KEYS" ]
@@ -87,6 +91,7 @@ function up-kops-cluster {
     $KOPS_BIN create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \
+    --container-runtime docker \
     --node-count 2 \
     --ssh-public-key=~/.ssh/devopsinuse.pub \
     --kubernetes-version ${K8S_VERSION} \
@@ -95,10 +100,13 @@ function up-kops-cluster {
     sleep 100
     $KOPS_BIN export kubeconfig --admin
     sleep 10
-    while [[ ! $($KOPS_BIN validate cluster | grep "is ready") ]]
+    MAX_RETRIES=15
+    RETRY_ATTEMPT=0
+    while [[ ! $($KOPS_BIN validate cluster | grep "is ready") && $RETRY_ATTEMPT -lt $MAX_RETRIES ]]
     do
-        sleep 5
-        echo "Waiting for cluster validation"
+        sleep 60
+        let RETRY_ATTEMPT=RETRY_ATTEMPT+1
+        echo "In attempt# $RETRY_ATTEMPT, waiting for cluster validation"
     done
     kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${MANIFEST_CNI_VERSION}/config/master/cni-metrics-helper.yaml
 }

diff --git a/scripts/lib/integration.sh b/scripts/lib/integration.sh
@@ -11,12 +11,10 @@ function run_kops_conformance() {
     done
     echo "Updated!"
 
-    GOPATH=$(go env GOPATH)
-    go install github.com/onsi/ginkgo/ginkgo
-    wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
+    wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test-linux-amd64.tar.gz | tar -zxvf - --strip-components=3 -C /tmp  kubernetes/test/bin/e2e.test
 
-    $GOPATH/bin/ginkgo -p --focus="Conformance"  --failFast --flakeAttempts 2 \
-    --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|(Basic StatefulSet functionality [StatefulSetBasic])|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
+    /tmp/e2e.test --ginkgo.focus="Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
+    --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|(Basic StatefulSet functionality [StatefulSetBasic])|\[Slow\]|\[Serial\]"
 
     /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
     --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
@@ -127,10 +127,11 @@ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --usernam
 ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_ECR_REPO_NAME"
 ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_INIT_ECR_REPO_NAME"
 
-# Check to see if the image already exists in the Docker repository, and if
+# Check to see if the image already exists in the ECR repository, and if
 # not, check out the CNI source code for that image tag, build the CNI
 # image and push it to the Docker repository
-if [[ $(docker images -q "$IMAGE_NAME:$TEST_IMAGE_VERSION" 2> /dev/null) ]]; then
+ecr_image_query_result=$(aws ecr batch-get-image --repository-name=amazon-k8s-cni --image-ids imageTag=$TEST_IMAGE_VERSION --query 'images[].imageId.imageTag' --region us-west-2)
+if [[ $ecr_image_query_result != "[]" ]]; then
     echo "CNI image $IMAGE_NAME:$TEST_IMAGE_VERSION already exists in repository. Skipping image build..."
     DOCKER_BUILD_DURATION=0
 else

diff --git a/scripts/test/config/perf-cluster.yml b/scripts/test/config/perf-cluster.yml
@@ -25,7 +25,7 @@ managedNodeGroups:
 
   - name: cni-test-multi-node-mng
     instanceType: m5.xlarge
-    desiredCapacity: 1
+    desiredCapacity: 99
     minSize: 1
     maxSize: 100
     iam:

diff --git a/test/framework/options.go b/test/framework/options.go
@@ -34,6 +34,8 @@ type Options struct {
 	NgNameLabelKey string
 	NgNameLabelVal string
 	EKSEndpoint    string
+	InitialAddon   string
+	TargetAddon    string
 }
 
 func (options *Options) BindFlags() {
@@ -44,6 +46,8 @@ func (options *Options) BindFlags() {
 	flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
 	flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
 	flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
+	flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied")
+	flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
 }
 
 func (options *Options) Validate() error {

diff --git a/test/framework/resources/aws/services/eks.go b/test/framework/resources/aws/services/eks.go
@@ -22,14 +22,32 @@ import (
 
 type EKS interface {
 	DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error)
-	CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error)
-	DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error)
+	CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error)
+	DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error)
+	DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error)
+	DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error)
+	GetLatestVersion(addonInput *AddonInput) (string, error)
 }
 
 type defaultEKS struct {
 	eksiface.EKSAPI
 }
 
+// Internal Addon Input struct
+// subset of eks.AddonInput
+// used by ginkgo tests
+type AddonInput struct {
+	AddonName    string
+	ClusterName  string
+	AddonVersion string
+	K8sVersion   string
+}
+
+type VersionWithBuild struct {
+	version uint
+	build   uint
+}
+
 func NewEKS(session *session.Session, endpoint string) EKS {
 	return &defaultEKS{
 		EKSAPI: eks.New(session, &aws.Config{
@@ -39,26 +57,54 @@ func NewEKS(session *session.Session, endpoint string) EKS {
 	}
 }
 
-func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) {
+func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) {
 	createAddonInput := &eks.CreateAddonInput{
-		AddonName:   aws.String(addon),
-		ClusterName: aws.String(clusterName),
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
+	}
+	if addonInput.AddonVersion != "" {
+		createAddonInput.SetAddonVersion(addonInput.AddonVersion)
+		createAddonInput.SetResolveConflicts("OVERWRITE")
 	}
 	return d.EKSAPI.CreateAddon(createAddonInput)
 }
 
-func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) {
+func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) {
 	deleteAddonInput := &eks.DeleteAddonInput{
-		AddonName:   aws.String(addon),
-		ClusterName: aws.String(clusterName),
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
 	}
 	return d.EKSAPI.DeleteAddon(deleteAddonInput)
 }
 
+func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) {
+	describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(addonInput.AddonName),
+		KubernetesVersion: aws.String(addonInput.K8sVersion),
+	}
+	return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput)
+}
+
+func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) {
+	describeAddonInput := &eks.DescribeAddonInput{
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
+	}
+	return d.EKSAPI.DescribeAddon(describeAddonInput)
+}
+
 func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) {
 	describeClusterInput := &eks.DescribeClusterInput{
 		Name: aws.String(clusterName),
 	}
 
 	return d.EKSAPI.DescribeCluster(describeClusterInput)
 }
+
+func (d defaultEKS) GetLatestVersion(addonInput *AddonInput) (string, error) {
+	addonOutput, err := d.DescribeAddonVersions(addonInput)
+	if err != nil {
+		return "", err
+	}
+	return *addonOutput.Addons[0].AddonVersions[0].AddonVersion, nil
+}
diff --git a/test/framework/resources/k8s/utils/addon.go b/test/framework/resources/k8s/utils/addon.go
@@ -0,0 +1,46 @@
+package utils
+
+import (
+	"context"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error {
+	ctx := context.Background()
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		_, err := eks.DescribeAddon(&services.AddonInput{
+			AddonName:   addonName,
+			ClusterName: clusterName,
+		})
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error {
+	ctx := context.Background()
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
+			AddonName:   addonName,
+			ClusterName: clusterName,
+		})
+		if err != nil {
+			return false, err
+		}
+
+		status := *describeAddonOutput.Addon.Status
+		if status == "CREATE_FAILED" || status == "DEGRADED" {
+			return false, errors.Errorf("Create Addon Failed, addon status: %s", status)
+		}
+		if status == "ACTIVE" {
+			return true, nil
+		}
+		return false, nil
+	}, ctx.Done())
+}