Patch Node InstanceGroup, set Node count

hack/e2e/kops.sh

@@ -22,12 +22,14 @@ function kops_create_cluster() {
   CLUSTER_NAME=${2}
   BIN=${3}
   ZONES=${4}
-  INSTANCE_TYPE=${5}
-  K8S_VERSION=${6}
-  CLUSTER_FILE=${7}
-  KUBECONFIG=${8}
-  KOPS_PATCH_FILE=${9}
-  KOPS_STATE_FILE=${10}
+  NODE_COUNT=${5}
+  INSTANCE_TYPE=${6}
+  K8S_VERSION=${7}
+  CLUSTER_FILE=${8}
+  KUBECONFIG=${9}
+  KOPS_PATCH_FILE=${10}
+  KOPS_PATCH_NODE_FILE=${11}
+  KOPS_STATE_FILE=${12}
 
   generate_ssh_key "${SSH_KEY_PATH}"
 
@@ -39,15 +41,18 @@ function kops_create_cluster() {
     ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
       --ssh-public-key="${SSH_KEY_PATH}".pub \
       --zones "${ZONES}" \
-      --node-count=3 \
+      --node-count="${NODE_COUNT}" \
       --node-size="${INSTANCE_TYPE}" \
       --kubernetes-version="${K8S_VERSION}" \
       --dry-run \
-      -o json \
+      -o yaml \
       "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
 
     if test -f "$KOPS_PATCH_FILE"; then
-      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE"
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
+    fi
+    if test -f "$KOPS_PATCH_NODE_FILE"; then
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node"
     fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
@@ -88,36 +93,63 @@ function kops_delete_cluster() {
   ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
 
-# TODO switch this to python or work exclusively with yaml, all this
-# hacking with jq stinks!
+# TODO switch this to python, work exclusively with yaml, use kops toolbox
+# template/kops set?, all this hacking with jq stinks!
 function kops_patch_cluster_file() {
-  CLUSTER_FILE=${1}    # input must be json
+  CLUSTER_FILE=${1}    # input must be yaml
   KOPS_PATCH_FILE=${2} # input must be yaml
+  KIND=${3}            # must be either Cluster or InstanceGroup
+  ROLE=${4}            # must be either Master or Node
 
   loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE"
 
-  # Temporary intermediate files for patching
+  # Temporary intermediate files for patching, don't mutate CLUSTER_FILE until
+  # the end
+  CLUSTER_FILE_JSON=$CLUSTER_FILE.json
   CLUSTER_FILE_0=$CLUSTER_FILE.0
   CLUSTER_FILE_1=$CLUSTER_FILE.1
 
-  # Output is an array of Cluster and InstanceGroups
-  jq '.[] | select(.kind=="Cluster")' "$CLUSTER_FILE" > "$CLUSTER_FILE_0"
+  # HACK convert the multiple yaml documents to an array of json objects
+  yaml_to_json "$CLUSTER_FILE" "$CLUSTER_FILE_JSON"
+
+  # Find the json objects to patch
+  FILTER=".[] | select(.kind==\"$KIND\")"
+  if [ -n "$ROLE" ]; then
+    FILTER="$FILTER | select(.spec.role==\"$ROLE\")"
+  fi
+  jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0"
 
-  # Patch only the Cluster
+  # Patch only the json objects
   kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
-  # Write the patched Cluster back to the array
-  jq '(.[] | select(.kind=="Cluster")) = $cluster[0]' "$CLUSTER_FILE" --slurpfile cluster "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
+  # Delete the original json objects, add the patched
+  # TODO Cluster must always be first?
+  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
-  # HACK convert the json array to multiple yaml documents
-  for ((i = 0; i < $(jq length "$CLUSTER_FILE_0"); i++)); do
-    echo "---" >> "$CLUSTER_FILE_1"
-    jq ".[$i]" "$CLUSTER_FILE_0" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$CLUSTER_FILE_1"
-  done
+  # HACK convert the array of json objects to multiple yaml documents
+  json_to_yaml "$CLUSTER_FILE_0" "$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
-  # Done patching, overwrite original CLUSTER_FILE
+  # Done patching, overwrite original yaml CLUSTER_FILE
   mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
+
+  # Clean up
+  rm "$CLUSTER_FILE_JSON"
+}
+
+function yaml_to_json() {
+  IN=${1}
+  OUT=${2}
+  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT"
+}
+
+function json_to_yaml() {
+  IN=${1}
+  OUT=${2}
+  for ((i = 0; i < $(jq length "$IN"); i++)); do
+    echo "---" >> "$OUT"
+    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
+  done
 }

hack/e2e/run.sh

@@ -35,12 +35,13 @@ CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
 TEST_DIR=${BASE_DIR}/csi-test-artifacts
 BIN_DIR=${TEST_DIR}/bin
 SSH_KEY_PATH=${TEST_DIR}/id_rsa
-CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.json
+CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
 KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
 
 REGION=${AWS_REGION:-us-west-2}
 ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
 FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
+NODE_COUNT=${NODE_COUNT:-3}
 INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large}
 
 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
@@ -54,6 +55,7 @@ K8S_VERSION=${K8S_VERSION:-1.20.6}
 KOPS_VERSION=${KOPS_VERSION:-1.20.0}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
+KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml}
 
 EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml}
 EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-}
@@ -112,11 +114,13 @@ if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
     "$CLUSTER_NAME" \
     "$KOPS_BIN" \
     "$ZONES" \
+    "$NODE_COUNT" \
     "$INSTANCE_TYPE" \
     "$K8S_VERSION" \
     "$CLUSTER_FILE" \
     "$KUBECONFIG" \
     "$KOPS_PATCH_FILE" \
+    "$KOPS_PATCH_NODE_FILE" \
     "$KOPS_STATE_FILE"
   if [[ $? -ne 0 ]]; then
     exit 1