Add integration_test job to CircleCI config

.circleci/config.yml

@@ -1,12 +1,16 @@
 # https://circleci.com/docs/2.0/language-go/
 version: 2.1
+
+orbs:
+  aws-cli: circleci/[email protected]
+
 jobs:
   build:
     docker:
       - image: circleci/golang:1.13-stretch
     working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
     environment:
-      GO111MODULE: "on"
+      GO111MODULE: 'on'
     steps:
       - checkout
       - run: go get -u golang.org/x/lint/golint
@@ -16,9 +20,30 @@ jobs:
       - run: make lint
       - run: make vet
       - run: make unit-test
-
+  integration_test:
+    docker:
+      - image: circleci/golang:1.13-stretch
+    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+    environment:
+      GO111MODULE: 'on'
+    steps:
+      - checkout
+      - setup_remote_docker
+      - aws-cli/setup:
+          profile-name: awstester
+      - run:
+          name: Run the integration tests
+          command: ./scripts/run-integration-tests.sh
+          no_output_timeout: 15m
+      - store_artifacts:
+          path: /tmp/cni-test
+
 workflows:
   version: 2
   check:
     jobs:
       - build
+      - integration_test:
+          context: aws
+          requires:
+            - build

.gitignore

@@ -9,4 +9,4 @@ verify-network
 .DS_Store
 cni-metrics-helper/cni-metrics-helper
 portmap
-grpc_health_probe
+grpc-health-probe

README.md

@@ -267,6 +267,28 @@ private IPs, which may be throttled, especially at scaling-related times.
 
 ---
 
+`MINIMUM_IP_TARGET`
+
+Type: Integer
+
+Default: None
+
+Specifies the number of total IP addresses that the `ipamD` daemon should attempt to allocate for pod assignment on the node.
+`MINIMUM_IP_TARGET` behaves identically to `WARM_IP_TARGET` except that instead of setting a target number of free IP
+addresses to keep available at all times, it sets a target number for a floor on how many total IP addresses are allocated.
+
+`MINIMUM_IP_TARGET` is for pre-scaling, `WARM_IP_TARGET` is for dynamic scaling. For example, suppose a cluster has an
+expected pod density of approximately 30 pods per node. If `WARM_IP_TARGET` is set to 30 to ensure there are enough IPs
+allocated up front by the CNI, then 30 pods are deployed to the node, the CNI will allocate an additional 30 IPs, for
+a total of 60, accelerating IP exhaustion in the relevant subnets. If instead `MINIMUM_IP_TARGET` is set to 30 and
+`WARM_IP_TARGET` to 2, after the 30 pods are deployed the CNI would allocate an additional 2 IPs. This still provides
+elasticity, but uses roughly half as many IPs as using WARM_IP_TARGET alone (32 IPs vs 60 IPs).
+
+This also improves reliability of the EKS cluster by reducing the number of calls necessary to allocate or deallocate
+private IPs, which may be throttled, especially at scaling-related times.
+
+---
+
 `MAX_ENI`
 
 Type: Integer

scripts/lib/aws.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 check_aws_credentials() {
-    aws sts get-caller-identity --query "Account" >/dev/null 2>&1 ||
+    aws sts get-caller-identity --query "Account" ||
         ( echo "No AWS credentials found. Please run \`aws configure\` to set up the CLI for your credentials." && exit 1)
 }
 

scripts/lib/cluster.sh

@@ -1,15 +1,19 @@
 #!/usr/bin/env bash
 
 function down-test-cluster() {
+    if [[ -n "${CIRCLE_JOB:-}" ]]; then
+        $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG || (echo "failed!" && exit 1)
+    else
     echo -n "Deleting cluster $CLUSTER_NAME (this may take ~10 mins) ... "
-    $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG >>$CLUSTER_MANAGE_LOG_PATH 2>&1 ||
-        ( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 )
+    $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG >> $CLUSTER_MANAGE_LOG_PATH 2>&1 ||
+        (echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1)
     echo "ok."
+    fi
 }
 
 function up-test-cluster() {
+    echo -n "Configuring cluster $CLUSTER_NAME"
     ssh-keygen -q -P cni-test -f $SSH_KEY_PATH
-
     AWS_K8S_TESTER_EKS_CLUSTER_NAME=$CLUSTER_NAME \
         AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \
         AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=${K8S_VERSION%.*} \
@@ -28,9 +32,14 @@ function up-test-cluster() {
         AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \
         AWS_K8S_TESTER_EKS_KUBECTL_PATH=$KUBECTL_PATH \
-    $TESTER_PATH eks create config --path $CLUSTER_CONFIG 1>&2
-    echo -n "Creating cluster $CLUSTER_NAME (this may take ~20 mins. details: tail -f $CLUSTER_MANAGE_LOG_PATH)... "
-    $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 2>>$CLUSTER_MANAGE_LOG_PATH 1>&2 ||
-        ( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 )
-    echo "ok."
+        $TESTER_PATH eks create config --path $CLUSTER_CONFIG 1>&2
+
+    if [[ -n "${CIRCLE_JOB:-}" ]]; then
+        $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG || (echo "failed!" && exit 1)
+    else
+        echo -n "Creating cluster $CLUSTER_NAME (this may take ~20 mins. details: tail -f $CLUSTER_MANAGE_LOG_PATH)... "
+        $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 2>> $CLUSTER_MANAGE_LOG_PATH 1>&2 ||
+            (echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1)
+        echo "ok."
+    fi
 }

scripts/run-integration-tests.sh

@@ -89,11 +89,11 @@ ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_ECR_REPO_NAME"
 # Check to see if the image already exists in the Docker repository, and if
 # not, check out the CNI source code for that image tag, build the CNI
 # image and push it to the Docker repository
-if [[ $(docker images -q $IMAGE_NAME:$TEST_IMAGE_VERSION 2>/dev/null) ]]; then
+if [[ $(docker images -q $IMAGE_NAME:$TEST_IMAGE_VERSION 2> /dev/null) ]]; then
     echo "CNI image $IMAGE_NAME:$TEST_IMAGE_VERSION already exists in repository. Skipping image build..."
 else
     echo "CNI image $IMAGE_NAME:$TEST_IMAGE_VERSION does not exist in repository."
-    if [[ $TEST_IMAGE_VERSION != $LOCAL_GIT_VERSION ]]; then
+    if [[ $TEST_IMAGE_VERSION != "$LOCAL_GIT_VERSION" ]]; then
         __cni_source_tmpdir="/tmp/cni-src-$IMAGE_VERSION"
         echo "Checking out CNI source code for $IMAGE_VERSION ..."
 
@@ -103,7 +103,7 @@ else
     fi
     make docker IMAGE=$IMAGE_NAME VERSION=$TEST_IMAGE_VERSION
     docker push $IMAGE_NAME:$TEST_IMAGE_VERSION
-    if [[ $TEST_IMAGE_VERSION != $LOCAL_GIT_VERSION ]]; then
+    if [[ $TEST_IMAGE_VERSION != "$LOCAL_GIT_VERSION" ]]; then
         popd
     fi
 fi
@@ -138,7 +138,9 @@ cp "$BASE_CONFIG_PATH" "$TEST_CONFIG_PATH"
 sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni,$IMAGE_NAME," "$TEST_CONFIG_PATH"
 sed -i'.bak' "s,$STABLE_IMAGE_VERSION,$TEST_IMAGE_VERSION," "$TEST_CONFIG_PATH"
 
+export KUBECONFIG=$KUBECONFIG_PATH
 ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
+
 echo "*******************************************************************************"
 echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE"
 echo ""
@@ -149,13 +151,12 @@ TEST_PASS=$?
 popd
 
 echo "*******************************************************************************"
-echo "Deploying CNI with image $IMAGE_NAME"
-export KUBECONFIG=$KUBECONFIG_PATH
-$KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
+echo "Updating CNI to image $IMAGE_NAME:$IMAGE_VERSION"
+$KUBECTL_PATH apply -f ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
 
-echo "Sleping for 90s"
+echo "Sleping for 110s"
 echo "TODO: Poll and wait for updates to complete instead!"
-sleep 90
+sleep 110
 
 echo "*******************************************************************************"
 echo "Running integration tests on current image:"