Patch kops yaml instead of appending to it

kubernetes-sigs · Apr 7, 2021 · 87e2426 · 87e2426
1 parent 1f76372
commit 87e2426
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 42 deletions.
diff --git a/hack/additional-policies.yaml b/hack/additional-policies.yaml
diff --git a/hack/e2e/kops.sh b/hack/e2e/kops.sh
@@ -26,37 +26,43 @@ function kops_create_cluster() {
   INSTANCE_TYPE=${6}
   K8S_VERSION=${7}
   TEST_DIR=${8}
-  KOPS_FEATURE_GATES_FILE=${10}
-  KOPS_ADDITIONAL_POLICIES_FILE=${11}
+  KOPS_PATCH_FILE=${10}
 
-  loudecho "Generating SSH key $SSH_KEY_PATH"
   if [[ ! -e ${SSH_KEY_PATH} ]]; then
+    loudecho "Generating SSH key $SSH_KEY_PATH"
     ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
+  else
+    loudecho "Reusing SSH key $SSH_KEY_PATH"
   fi
 
+  CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.json
+
   set +e
   if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
     set -e
-    loudecho "Updating cluster $CLUSTER_NAME"
+    loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   else
     set -e
-    loudecho "Creating cluster $CLUSTER_NAME"
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
     ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
       --zones "${ZONES}" \
       --node-count=3 \
       --node-size="${INSTANCE_TYPE}" \
       --kubernetes-version="${K8S_VERSION}" \
-      --ssh-public-key="${SSH_KEY_PATH}".pub \
-      "${CLUSTER_NAME}"
+      --dry-run \
+      -o json \
+      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+
+    kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE"
+
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   fi
 
-  CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
-  ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
-  # TODO this appends duplicate values
-  [ -r "$KOPS_FEATURE_GATES_FILE" ] && cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
-  [ -r "$KOPS_ADDITIONAL_POLICIES_FILE" ] && cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
-  ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
-  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
+  loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE"
+  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" \
+    --ssh-public-key="${SSH_KEY_PATH}".pub --yes
 
   ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin
 
@@ -72,3 +78,36 @@ function kops_delete_cluster() {
   loudecho "Deleting cluster ${CLUSTER_NAME}"
   ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
+
+# TODO switch this to python, all this hacking with jq stinks!
+function kops_patch_cluster_file() {
+  CLUSTER_FILE=${1}
+  KOPS_PATCH_FILE=${2}
+
+  loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE"
+
+  # Temporary intermediate files for patching
+  CLUSTER_FILE_0=$CLUSTER_FILE.0
+  CLUSTER_FILE_1=$CLUSTER_FILE.1
+
+  # Output is an array of Cluster and InstanceGroups
+  jq '.[] | select(.kind=="Cluster")' "$CLUSTER_FILE" > "$CLUSTER_FILE_0"
+
+  # Patch only the Cluster
+  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # Write the patched Cluster back to the array
+  jq '(.[] | select(.kind=="Cluster")) = $cluster[0]' "$CLUSTER_FILE" --slurpfile cluster "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # HACK convert the json array to multiple yaml documents
+  for ((i = 0; i < $(jq length "$CLUSTER_FILE_0"); i++)); do
+    echo "---" >> "$CLUSTER_FILE_1"
+    jq ".[$i]" "$CLUSTER_FILE_0" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$CLUSTER_FILE_1"
+  done
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # Done patching, overwrite original CLUSTER_FILE
+  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE"
+}
diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh
@@ -46,8 +46,7 @@ IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}}
 K8S_VERSION=${K8S_VERSION:-1.20.4}
 KOPS_VERSION=${KOPS_VERSION:-1.20.0-beta.2}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
-KOPS_FEATURE_GATES_FILE=${KOPS_FEATURE_GATES_FILE:-./hack/feature-gates.yaml}
-KOPS_ADDITIONAL_POLICIES_FILE=${KOPS_ADDITIONAL_POLICIES_FILE:-./hack/additional-policies.yaml}
+KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
 
 HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml}
 
@@ -99,8 +98,7 @@ kops_create_cluster \
   "$K8S_VERSION" \
   "$TEST_DIR" \
   "$BASE_DIR" \
-  "$KOPS_FEATURE_GATES_FILE" \
-  "$KOPS_ADDITIONAL_POLICIES_FILE"
+  "$KOPS_PATCH_FILE"
 if [[ $? -ne 0 ]]; then
   exit 1
 fi

diff --git a/hack/feature-gates.yaml → hack/kops-patch.yaml b/hack/feature-gates.yaml → hack/kops-patch.yaml
@@ -1,3 +1,28 @@
+spec:
+  additionalPolicies:
+    node: |
+      [
+        {
+          "Effect": "Allow",
+          "Action": [
+            "ec2:AttachVolume",
+            "ec2:CreateSnapshot",
+            "ec2:CreateTags",
+            "ec2:CreateVolume",
+            "ec2:DeleteSnapshot",
+            "ec2:DeleteTags",
+            "ec2:DeleteVolume",
+            "ec2:DescribeInstances",
+            "ec2:DescribeSnapshots",
+            "ec2:DescribeTags",
+            "ec2:DescribeVolumes",
+            "ec2:DetachVolume",
+            "ec2:ModifyVolume",
+            "ec2:DescribeVolumesModifications"
+          ],
+          "Resource": "*"
+        }
+      ]
   kubeAPIServer:
     featureGates:
       CSIDriverRegistry: "true"
@@ -26,3 +51,4 @@
       CSIMigrationAWS: "true"
       ExpandCSIVolumes: "true"
       CSIInlineVolume: "true"
+    anonymousAuth: false