diff --git a/Makefile b/Makefile index 21a3c630bc..91781e98c7 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,7 @@ test-e2e-external-eks: CLUSTER_TYPE=eksctl \ K8S_VERSION="1.20" \ HELM_VALUES_FILE="./hack/values_eksctl.yaml" \ + EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \ AWS_REGION=us-west-2 \ AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \ TEST_PATH=./tests/e2e-kubernetes/... \ diff --git a/hack/e2e/ecr.sh b/hack/e2e/ecr.sh index 8f710d95be..df7b973b71 100644 --- a/hack/e2e/ecr.sh +++ b/hack/e2e/ecr.sh @@ -11,7 +11,7 @@ function ecr_build_and_push() { IMAGE_NAME=${3} IMAGE_TAG=${4} set +e - if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then + if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then set -e loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed" else diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl.sh index e069ec9cd8..a4c7f2011b 100644 --- a/hack/e2e/eksctl.sh +++ b/hack/e2e/eksctl.sh @@ -21,6 +21,7 @@ function eksctl_create_cluster() { CLUSTER_FILE=${7} KUBECONFIG=${8} EKSCTL_PATCH_FILE=${9} + EKSCTL_ADMIN_ROLE=${10} generate_ssh_key "${SSH_KEY_PATH}" @@ -55,6 +56,14 @@ function eksctl_create_cluster() { loudecho "Getting cluster ${CLUSTER_NAME}" ${BIN} get cluster "${CLUSTER_NAME}" + + if [ -n "$EKSCTL_ADMIN_ROLE" ]; then + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}" + loudecho "Granting ${ADMIN_ARN} admin access to the cluster" + ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin + fi + return $? } diff --git a/hack/e2e/kops.sh b/hack/e2e/kops.sh index f7fcb487a4..ef12d5ba56 100644 --- a/hack/e2e/kops.sh +++ b/hack/e2e/kops.sh @@ -22,12 +22,14 @@ function kops_create_cluster() { CLUSTER_NAME=${2} BIN=${3} ZONES=${4} - INSTANCE_TYPE=${5} - K8S_VERSION=${6} - CLUSTER_FILE=${7} - KUBECONFIG=${8} - KOPS_PATCH_FILE=${9} - KOPS_STATE_FILE=${10} + NODE_COUNT=${5} + INSTANCE_TYPE=${6} + K8S_VERSION=${7} + CLUSTER_FILE=${8} + KUBECONFIG=${9} + KOPS_PATCH_FILE=${10} + KOPS_PATCH_NODE_FILE=${11} + KOPS_STATE_FILE=${12} generate_ssh_key "${SSH_KEY_PATH}" @@ -39,15 +41,18 @@ function kops_create_cluster() { ${BIN} create cluster --state "${KOPS_STATE_FILE}" \ --ssh-public-key="${SSH_KEY_PATH}".pub \ --zones "${ZONES}" \ - --node-count=3 \ + --node-count="${NODE_COUNT}" \ --node-size="${INSTANCE_TYPE}" \ --kubernetes-version="${K8S_VERSION}" \ --dry-run \ - -o json \ + -o yaml \ "${CLUSTER_NAME}" > "${CLUSTER_FILE}" if test -f "$KOPS_PATCH_FILE"; then - kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" + kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" "" + fi + if test -f "$KOPS_PATCH_NODE_FILE"; then + kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node" fi loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" @@ -88,36 +93,63 @@ function kops_delete_cluster() { ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes } -# TODO switch this to python or work exclusively with yaml, all this -# hacking with jq stinks! +# TODO switch this to python, work exclusively with yaml, use kops toolbox +# template/kops set?, all this hacking with jq stinks! function kops_patch_cluster_file() { - CLUSTER_FILE=${1} # input must be json + CLUSTER_FILE=${1} # input must be yaml KOPS_PATCH_FILE=${2} # input must be yaml + KIND=${3} # must be either Cluster or InstanceGroup + ROLE=${4} # must be either Master or Node loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE" - # Temporary intermediate files for patching + # Temporary intermediate files for patching, don't mutate CLUSTER_FILE until + # the end + CLUSTER_FILE_JSON=$CLUSTER_FILE.json CLUSTER_FILE_0=$CLUSTER_FILE.0 CLUSTER_FILE_1=$CLUSTER_FILE.1 - # Output is an array of Cluster and InstanceGroups - jq '.[] | select(.kind=="Cluster")' "$CLUSTER_FILE" > "$CLUSTER_FILE_0" + # HACK convert the multiple yaml documents to an array of json objects + yaml_to_json "$CLUSTER_FILE" "$CLUSTER_FILE_JSON" + + # Find the json objects to patch + FILTER=".[] | select(.kind==\"$KIND\")" + if [ -n "$ROLE" ]; then + FILTER="$FILTER | select(.spec.role==\"$ROLE\")" + fi + jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0" - # Patch only the Cluster + # Patch only the json objects kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" - # Write the patched Cluster back to the array - jq '(.[] | select(.kind=="Cluster")) = $cluster[0]' "$CLUSTER_FILE" --slurpfile cluster "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1" + # Delete the original json objects, add the patched + # TODO Cluster must always be first? + jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" - # HACK convert the json array to multiple yaml documents - for ((i = 0; i < $(jq length "$CLUSTER_FILE_0"); i++)); do - echo "---" >> "$CLUSTER_FILE_1" - jq ".[$i]" "$CLUSTER_FILE_0" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$CLUSTER_FILE_1" - done + # HACK convert the array of json objects to multiple yaml documents + json_to_yaml "$CLUSTER_FILE_0" "$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" - # Done patching, overwrite original CLUSTER_FILE + # Done patching, overwrite original yaml CLUSTER_FILE mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml + + # Clean up + rm "$CLUSTER_FILE_JSON" +} + +function yaml_to_json() { + IN=${1} + OUT=${2} + kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT" +} + +function json_to_yaml() { + IN=${1} + OUT=${2} + for ((i = 0; i < $(jq length "$IN"); i++)); do + echo "---" >> "$OUT" + jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT" + done } diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index fd1c7fef2f..27035041e7 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -35,12 +35,13 @@ CLUSTER_TYPE=${CLUSTER_TYPE:-kops} TEST_DIR=${BASE_DIR}/csi-test-artifacts BIN_DIR=${TEST_DIR}/bin SSH_KEY_PATH=${TEST_DIR}/id_rsa -CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.json +CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"} REGION=${AWS_REGION:-us-west-2} ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1) +NODE_COUNT=${NODE_COUNT:-3} INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large} AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) @@ -54,8 +55,10 @@ K8S_VERSION=${K8S_VERSION:-1.20.6} KOPS_VERSION=${KOPS_VERSION:-1.20.0} KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e} KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml} +KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml} EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml} +EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-} HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml} @@ -111,11 +114,13 @@ if [[ "${CLUSTER_TYPE}" == "kops" ]]; then "$CLUSTER_NAME" \ "$KOPS_BIN" \ "$ZONES" \ + "$NODE_COUNT" \ "$INSTANCE_TYPE" \ "$K8S_VERSION" \ "$CLUSTER_FILE" \ "$KUBECONFIG" \ "$KOPS_PATCH_FILE" \ + "$KOPS_PATCH_NODE_FILE" \ "$KOPS_STATE_FILE" if [[ $? -ne 0 ]]; then exit 1 @@ -130,7 +135,8 @@ elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then "$K8S_VERSION" \ "$CLUSTER_FILE" \ "$KUBECONFIG" \ - "$EKSCTL_PATCH_FILE" + "$EKSCTL_PATCH_FILE" \ + "$EKSCTL_ADMIN_ROLE" if [[ $? -ne 0 ]]; then exit 1 fi