Fast-forward to latest ebs hack/e2e scripts with eksctl support, k8s …

…1.20, etc.

hack/e2e/ebs.sh

@@ -6,11 +6,13 @@ BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 source "${BASE_DIR}"/util.sh
 
 function ebs_check_migration() {
+  KUBECONFIG=${1}
+
   loudecho "Checking migration"
   # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
   # Find the controller-manager log and read its metrics to verify
-  NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
-  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &
+  NODE=$(kubectl get node -l kubernetes.io/role=master -o json --kubeconfig "${KUBECONFIG}" | jq -r ".items[].metadata.name")
+  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system --kubeconfig "${KUBECONFIG}" &
 
   # Ensure port forwarding succeeded
   n=0

hack/e2e/ecr.sh

@@ -11,7 +11,7 @@ function ecr_build_and_push() {
   IMAGE_NAME=${3}
   IMAGE_TAG=${4}
   set +e
-  if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
+  if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then
     set -e
     loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
   else

hack/e2e/eksctl.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+set -euo pipefail
+
+function eksctl_install() {
+  INSTALL_PATH=${1}
+  EKSCTL_VERSION=${2}
+  if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then
+    EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz"
+    curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
+    chmod +x "${INSTALL_PATH}"/eksctl
+  fi
+}
+
+function eksctl_create_cluster() {
+  SSH_KEY_PATH=${1}
+  CLUSTER_NAME=${2}
+  BIN=${3}
+  ZONES=${4}
+  INSTANCE_TYPE=${5}
+  K8S_VERSION=${6}
+  CLUSTER_FILE=${7}
+  KUBECONFIG=${8}
+  EKSCTL_PATCH_FILE=${9}
+  EKSCTL_ADMIN_ROLE=${10}
+
+  generate_ssh_key "${SSH_KEY_PATH}"
+
+  CLUSTER_NAME="${CLUSTER_NAME//./-}"
+
+  if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then
+    loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${BIN} upgrade cluster -f "${CLUSTER_FILE}"
+  else
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
+    ${BIN} create cluster \
+      --managed \
+      --ssh-access \
+      --ssh-public-key "${SSH_KEY_PATH}".pub \
+      --zones "${ZONES}" \
+      --nodes=3 \
+      --instance-types="${INSTANCE_TYPE}" \
+      --version="${K8S_VERSION}" \
+      --disable-pod-imds \
+      --dry-run \
+      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+
+    if test -f "$EKSCTL_PATCH_FILE"; then
+      eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE"
+    fi
+
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
+  fi
+
+  loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"
+
+  loudecho "Getting cluster ${CLUSTER_NAME}"
+  ${BIN} get cluster "${CLUSTER_NAME}"
+
+  if [ -n "$EKSCTL_ADMIN_ROLE" ]; then
+    AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+    ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}"
+    loudecho "Granting ${ADMIN_ARN} admin access to the cluster"
+    ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
+  fi
+
+  return $?
+}
+
+function eksctl_cluster_exists() {
+  CLUSTER_NAME=${1}
+  BIN=${2}
+  set +e
+  if ${BIN} get cluster "${CLUSTER_NAME}"; then
+    set -e
+    return 0
+  else
+    set -e
+    return 1
+  fi
+}
+
+function eksctl_delete_cluster() {
+  BIN=${1}
+  CLUSTER_NAME=${2}
+  loudecho "Deleting cluster ${CLUSTER_NAME}"
+  ${BIN} delete cluster "${CLUSTER_NAME}"
+}
+
+function eksctl_patch_cluster_file() {
+  CLUSTER_FILE=${1}      # input must be yaml
+  EKSCTL_PATCH_FILE=${2} # input must be yaml
+
+  loudecho "Patching cluster $CLUSTER_NAME with $EKSCTL_PATCH_FILE"
+
+  # Temporary intermediate files for patching
+  CLUSTER_FILE_0=$CLUSTER_FILE.0
+  CLUSTER_FILE_1=$CLUSTER_FILE.1
+
+  cp "$CLUSTER_FILE" "$CLUSTER_FILE_0"
+
+  # Patch only the Cluster
+  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # Done patching, overwrite original CLUSTER_FILE
+  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
+}

hack/e2e/kops.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -uo pipefail
+set -euo pipefail
 
 OS_ARCH=$(go env GOOS)-amd64
 
@@ -19,53 +19,137 @@ function kops_install() {
 
 function kops_create_cluster() {
   SSH_KEY_PATH=${1}
-  KOPS_STATE_FILE=${2}
-  CLUSTER_NAME=${3}
-  KOPS_BIN=${4}
-  ZONES=${5}
+  CLUSTER_NAME=${2}
+  BIN=${3}
+  ZONES=${4}
+  NODE_COUNT=${5}
   INSTANCE_TYPE=${6}
   K8S_VERSION=${7}
-  TEST_DIR=${8}
-  KOPS_FEATURE_GATES_FILE=${10}
-  KOPS_ADDITIONAL_POLICIES_FILE=${11}
+  CLUSTER_FILE=${8}
+  KUBECONFIG=${9}
+  KOPS_PATCH_FILE=${10}
+  KOPS_PATCH_NODE_FILE=${11}
+  KOPS_STATE_FILE=${12}
 
-  loudecho "Generating SSH key $SSH_KEY_PATH"
-  if [[ ! -e ${SSH_KEY_PATH} ]]; then
-    ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
-  fi
+  generate_ssh_key "${SSH_KEY_PATH}"
 
-  set +e
-  if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
-    set -e
-    loudecho "Updating cluster $CLUSTER_NAME"
+  if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
+    loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   else
-    set -e
-    loudecho "Creating cluster $CLUSTER_NAME"
-    ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
+    ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
+      --ssh-public-key="${SSH_KEY_PATH}".pub \
       --zones "${ZONES}" \
-      --node-count=3 \
+      --node-count="${NODE_COUNT}" \
       --node-size="${INSTANCE_TYPE}" \
       --kubernetes-version="${K8S_VERSION}" \
-      --ssh-public-key="${SSH_KEY_PATH}".pub \
-      "${CLUSTER_NAME}"
+      --dry-run \
+      -o yaml \
+      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+
+    if test -f "$KOPS_PATCH_FILE"; then
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
+    fi
+    if test -f "$KOPS_PATCH_NODE_FILE"; then
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node"
+    fi
+
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    kops create secret --state "${KOPS_STATE_FILE}" --name "${CLUSTER_NAME}" sshpublickey admin -i "${SSH_KEY_PATH}".pub
   fi
 
-  CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
-  ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
-  [ -r "$KOPS_FEATURE_GATES_FILE" ] && cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
-  [ -r "$KOPS_ADDITIONAL_POLICIES_FILE" ] && cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
-  ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
-  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
+  loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE"
+  ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
 
-  loudecho "Validating cluster $CLUSTER_NAME"
-  ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
+  loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}"
+  ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
+
+  loudecho "Validating cluster ${CLUSTER_NAME}"
+  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
   return $?
 }
 
+function kops_cluster_exists() {
+  CLUSTER_NAME=${1}
+  BIN=${2}
+  KOPS_STATE_FILE=${3}
+  set +e
+  if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
+    set -e
+    return 0
+  else
+    set -e
+    return 1
+  fi
+}
+
 function kops_delete_cluster() {
-  KOPS_BIN=${1}
+  BIN=${1}
   CLUSTER_NAME=${2}
   KOPS_STATE_FILE=${3}
   loudecho "Deleting cluster ${CLUSTER_NAME}"
-  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+  ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+}
+
+# TODO switch this to python, work exclusively with yaml, use kops toolbox
+# template/kops set?, all this hacking with jq stinks!
+function kops_patch_cluster_file() {
+  CLUSTER_FILE=${1}    # input must be yaml
+  KOPS_PATCH_FILE=${2} # input must be yaml
+  KIND=${3}            # must be either Cluster or InstanceGroup
+  ROLE=${4}            # must be either Master or Node
+
+  loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE"
+
+  # Temporary intermediate files for patching, don't mutate CLUSTER_FILE until
+  # the end
+  CLUSTER_FILE_JSON=$CLUSTER_FILE.json
+  CLUSTER_FILE_0=$CLUSTER_FILE.0
+  CLUSTER_FILE_1=$CLUSTER_FILE.1
+
+  # HACK convert the multiple yaml documents to an array of json objects
+  yaml_to_json "$CLUSTER_FILE" "$CLUSTER_FILE_JSON"
+
+  # Find the json objects to patch
+  FILTER=".[] | select(.kind==\"$KIND\")"
+  if [ -n "$ROLE" ]; then
+    FILTER="$FILTER | select(.spec.role==\"$ROLE\")"
+  fi
+  jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0"
+
+  # Patch only the json objects
+  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # Delete the original json objects, add the patched
+  # TODO Cluster must always be first?
+  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # HACK convert the array of json objects to multiple yaml documents
+  json_to_yaml "$CLUSTER_FILE_0" "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # Done patching, overwrite original yaml CLUSTER_FILE
+  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
+
+  # Clean up
+  rm "$CLUSTER_FILE_JSON"
+}
+
+function yaml_to_json() {
+  IN=${1}
+  OUT=${2}
+  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT"
+}
+
+function json_to_yaml() {
+  IN=${1}
+  OUT=${2}
+  for ((i = 0; i < $(jq length "$IN"); i++)); do
+    echo "---" >> "$OUT"
+    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
+  done
 }