Skip to content

Commit

Permalink
Changes to run e2e for private cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
jigisha620 committed Apr 1, 2024
1 parent f967e70 commit dc925db
Show file tree
Hide file tree
Showing 25 changed files with 638 additions and 99 deletions.
4 changes: 4 additions & 0 deletions .github/actions/e2e/cleanup/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ inputs:
eksctl_version:
description: "Version of eksctl to install"
default: v0.169.0
private_cluster:
description: "Whether the cluster that has to be deleted is private or not. Valid values are 'true' or 'false'"
default: 'false'
runs:
using: "composite"
steps:
Expand All @@ -28,6 +31,7 @@ runs:
with:
version: ${{ inputs.eksctl_version }}
- name: delete-cluster
if: ${{ inputs.private_cluster == 'false' }}
shell: bash
env:
CLUSTER_NAME: ${{ inputs.cluster_name }}
Expand Down
7 changes: 2 additions & 5 deletions .github/actions/e2e/install-helm/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,9 @@ runs:
- name: install helm
shell: bash
env:
VERSION: ${{ inputs.version }}
HELM_VERSION: ${{ inputs.version }}
run: |
TEMPDIR=$(mktemp -d)
curl -fsSL -o "${TEMPDIR}/get_helm.sh" https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 "${TEMPDIR}/get_helm.sh"
"${TEMPDIR}/get_helm.sh" --version "$VERSION"
./test/hack/e2e_scripts/install_helm.sh
- name: install helm-diff
shell: bash
run: |
Expand Down
44 changes: 6 additions & 38 deletions .github/actions/e2e/install-karpenter/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ inputs:
default: "1.29"
git_ref:
description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release"
private_cluster:
description: "Whether the cluster is private or not. Valid values are 'true' or 'false'"
default: 'false'
runs:
using: "composite"
steps:
Expand Down Expand Up @@ -53,48 +56,13 @@ runs:
ACCOUNT_ID: ${{ inputs.account_id }}
CLUSTER_NAME: ${{ inputs.cluster_name }}
K8S_VERSION: ${{ inputs.k8s_version }}
PRIVATE_CLUSTER: ${{ inputs.private_cluster }}
run: |
aws eks update-kubeconfig --name "$CLUSTER_NAME"
# Parse minor version to determine whether to enable the webhooks
K8S_VERSION_MINOR="${K8S_VERSION#*.}"
WEBHOOK_ENABLED=false
if (( K8S_VERSION_MINOR < 25 )); then
WEBHOOK_ENABLED=true
fi
# Remove service account annotation when dropping support for 1.23
helm upgrade --install karpenter "oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" \
-n kube-system \
--version "0-$(git rev-parse HEAD)" \
--set logLevel=debug \
--set webhook.enabled=${WEBHOOK_ENABLED} \
--set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \
--set settings.clusterName="$CLUSTER_NAME" \
--set settings.interruptionQueue="$CLUSTER_NAME" \
--set settings.featureGates.spotToSpotConsolidation=true \
--set controller.resources.requests.cpu=3 \
--set controller.resources.requests.memory=3Gi \
--set controller.resources.limits.cpu=3 \
--set controller.resources.limits.memory=3Gi \
--set serviceMonitor.enabled=true \
--set serviceMonitor.additionalLabels.scrape=enabled \
--set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \
--set "serviceMonitor.endpointConfig.relabelings[0].replacement=$CLUSTER_NAME" \
--set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \
--set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \
--set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \
--set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \
--set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \
--set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \
--wait
./test/hack/e2e_scripts/install_karpenter.sh
- name: diff-karpenter
shell: bash
env:
ECR_ACCOUNT_ID: ${{ inputs.ecr_account_id }}
ECR_REGION: ${{ inputs.ecr_region }}
run: |
helm diff upgrade --namespace kube-system \
karpenter oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter \
--version 0-$(git rev-parse HEAD) \
--reuse-values --three-way-merge --detailed-exitcode
./test/hack/e2e_scripts/diff_karpenter.sh
28 changes: 9 additions & 19 deletions .github/actions/e2e/install-prometheus/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ inputs:
region:
description: "Region to access AWS"
required: true
prometheus_region:
description: "Prometheus region"
required: true
cluster_name:
description: 'Name of the cluster to be launched by eksctl'
required: true
Expand All @@ -18,6 +21,9 @@ inputs:
required: true
git_ref:
description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release"
private_cluster:
description: "Whether the cluster is private or not. Valid values are 'true' or 'false'"
default: 'false'
runs:
using: "composite"
steps:
Expand All @@ -39,27 +45,11 @@ runs:
- name: install prometheus
shell: bash
env:
PROMETHEUS_REGION: ${{ inputs.prometheus_region }}
REGION: ${{ inputs.region }}
WORKSPACE_ID: ${{ inputs.workspace_id }}
ACCOUNT_ID: ${{ inputs.account_id }}
CLUSTER_NAME: ${{ inputs.cluster_name }}
PRIVATE_CLUSTER: ${{ inputs.private_cluster }}
run: |
# Remove service account annotation when dropping support for 1.23
helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \
-n prometheus \
-f ./.github/actions/e2e/install-prometheus/values.yaml \
--set prometheus.prometheusSpec.remoteWrite[0].url=https://aps-workspaces.$REGION.amazonaws.com/workspaces/$WORKSPACE_ID/api/v1/remote_write \
--set prometheus.prometheusSpec.remoteWrite[0].sigv4.region=$REGION \
--set prometheus.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/prometheus-irsa-$CLUSTER_NAME" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[1].targetLabel=clusterName" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[1].replacement=$CLUSTER_NAME" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[2].targetLabel=gitRef" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[2].replacement=$(git rev-parse HEAD)" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[3].targetLabel=mostRecentTag" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[3].replacement=$(git describe --abbrev=0 --tags)" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[4].targetLabel=commitsAfterTag" \
--set "kubelet.serviceMonitor.cAdvisorRelabelings[4].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \
--wait
./test/hack/e2e_scripts/install_prometheus.sh
159 changes: 159 additions & 0 deletions .github/actions/e2e/run-tests-private-cluster/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
name: RunTestsPrivateCluster
description: 'Installs Karpenter, Prometheus, runs tests on private cluster and performs clean up'
inputs:
account_id:
description: "Account ID to access AWS"
required: true
suite:
type: string
required: true
ecr_account_id:
description: "Account ID to access ECR Repository"
required: true
prometheus_workspace_id:
description: "Workspace ID for the Prometheus workspace"
required: true
metrics_region:
description: "Metrics region"
required: true
node_role:
description: "Private cluster node role"
required: true
region:
description: "Region to access AWS"
required: true
ecr_region:
description: "Region to access ECR Repository"
required: true
prometheus_region:
description: Region to access Prometheus
required: true
cluster_name:
description: 'Name of the cluster to be launched by eksctl'
required: true
k8s_version:
description: 'Version of Kubernetes to use for the launched cluster'
default: "1.29"
private_cluster:
description: "Whether to create a private cluster which does not add access to the public internet. Valid values are 'true' or 'false'"
default: 'false'
enable_metrics:
description: "Whether to enable metrics for the cluster"
default: 'false'
codebuild_sg:
description: "Codebuild security group to run private cluster tests"
required: true
codebuild_vpc:
description: "Codebuild VPC to run private cluster tests"
required: true
cleanup:
description: "Whether to cleanup resources on failure"
default: 'false'
runs:
using: "composite"
steps:
- name: login to ecr via docker
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ${{ inputs.account_id }}.dkr.ecr.${{ inputs.region }}.amazonaws.com
logout: true
- name: configure private cluster
if: ${{ inputs.private_cluster }}
shell: bash
env:
REGION: ${{ inputs.region }}
CLUSTER_NAME: ${{ inputs.cluster_name }}
ACCOUNT_ID: ${{ inputs.account_id }}
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
CODEBUILD_SG: ${{ inputs.codebuild_sg }}
CODEBUILD_VPC: ${{ inputs.codebuild_vpc }}
run: |
./test/hack/e2e_scripts/configure_private_cluster.sh
- name: run private cluster tests on codebuild
env:
SUITE: ${{ inputs.suite }}
CLUSTER_NAME: ${{ inputs.cluster_name }}
INTERRUPTION_QUEUE: ${{ inputs.cluster_name }}
REGION: ${{ inputs.region }}
HELM_VERSION: v3.12.3 # Pinned to this version since v3.13.0 has issues with anonymous pulls: https://github.com/helm/helm/issues/12423
PROMETHEUS_REGION: ${{ inputs.prometheus_region }}
WORKSPACE_ID: ${{ inputs.prometheus_workspace_id }}
ACCOUNT_ID: ${{ inputs.account_id }}
K8S_VERSION: ${{ inputs.k8s_version }}
ECR_ACCOUNT_ID: ${{ inputs.ecr_account_id }}
ECR_REGION: ${{ inputs.ecr_region }}
PRIVATE_CLUSTER: ${{ inputs.private_cluster }}
ENABLE_METRICS: ${{ inputs.enable_metrics }}
METRICS_REGION: ${{ inputs.metrics_region }}
VPC_PEERING_CONNECTION_ID: ${{ env.VPC_PEERING_CONNECTION_ID }}
NODE_ROLE: ${{ env.NODE_ROLE }}
SG_CB: ${{ inputs.codebuild_sg }}
VPC_CB: ${{ inputs.codebuild_vpc }}
CLUSTER_VPC_ID: ${{ env.CLUSTER_VPC_ID }}
EKS_CLUSTER_SG: ${{ env.EKS_CLUSTER_SG }}
CLEANUP: ${{ inputs.cleanup }}
uses: aws-actions/aws-codebuild-run-build@bafa4d8b0d8802b5adf3a54861f530792d2e4f24 #v1.0.15
with:
project-name: E2EPrivateClusterCodeBuildProject-us-east-1
buildspec-override: |
version: 0.2
phases:
install:
commands:
# Make sure goenv is up to date
- cd $HOME/.goenv && git pull --ff-only && cd -
# Install Go 1.22
- goenv install 1.22 && goenv global 1.22
build:
commands:
- aws eks update-kubeconfig --name $CLUSTER_NAME
- ./test/hack/e2e_scripts/noderole_bootstrap_permission.sh
- ./test/hack/e2e_scripts/install_helm.sh
- helm plugin install https://github.com/databus23/helm-diff || true
- aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com
- helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
- helm pull prometheus-community/kube-prometheus-stack
- kubectl create ns prometheus || true
- kubectl label ns prometheus scrape=enabled --overwrite=true
- ./test/hack/e2e_scripts/install_prometheus.sh
- kubectl label ns kube-system scrape=enabled --overwrite=true
- kubectl label ns kube-system pod-security.kubernetes.io/warn=restricted --overwrite=true
- ./test/hack/e2e_scripts/install_karpenter.sh
- ./test/hack/e2e_scripts/diff_karpenter.sh
- kubectl delete nodepool --all
- kubectl delete ec2nodeclass --all
- kubectl delete deployment --all
- PRIVATE_CLUSTER=$CLUSTER_NAME TEST_SUITE=$SUITE ENABLE_METRICS=$ENABLE_METRICS METRICS_REGION=$METRICS_REGION GIT_REF="$(git rev-parse HEAD)" CLUSTER_NAME=$CLUSTER_NAME CLUSTER_ENDPOINT="$(aws eks describe-cluster --name $CLUSTER_NAME --query "cluster.endpoint" --output text)" INTERRUPTION_QUEUE=$CLUSTER_NAME make e2etests
post_build:
commands:
# Describe karpenter pods
- kubectl describe pods -n kube-system -l app.kubernetes.io/name=karpenter
# Describe nodes
- kubectl describe nodes
- |
if [ "${CLEANUP}" = true ]; then
./test/hack/e2e_scripts/clean_private_cluster.sh
fi
env-vars-for-codebuild: |
SUITE,
CLUSTER_NAME,
INTERRUPTION_QUEUE,
REGION,
HELM_VERSION,
PROMETHEUS_REGION,
WORKSPACE_ID,
ACCOUNT_ID,
K8S_VERSION,
ECR_ACCOUNT_ID,
ECR_REGION,
PRIVATE_CLUSTER,
ENABLE_METRICS,
METRICS_REGION,
VPC_PEERING_CONNECTION_ID,
NODE_ROLE,
SG_CB,
VPC_CB,
CLUSTER_VPC_ID,
EKS_CLUSTER_SG,
CLEANUP
Loading

0 comments on commit dc925db

Please sign in to comment.