ci: Remove Service account annotation for Karpenter and Prometheus

.github/actions/e2e/install-karpenter/action.yaml

@@ -33,12 +33,13 @@ runs:
   - uses: ./.github/actions/e2e/install-helm
     with:
       version: v3.12.3 # Pinned to this version since v3.13.0 has issues with anonymous pulls: https://github.com/helm/helm/issues/12423
-  - name: create karpenter namespace
+  # Label namespace to enforce security stranded and scrape prometheus metrics 
+  # https://kubernetes.io/docs/concepts/security/pod-security-standards/
+  - name: add labels to kube-system namespace
     shell: bash
     run: |
-      kubectl create ns karpenter || true
-      kubectl label ns karpenter scrape=enabled --overwrite=true
-      kubectl label ns karpenter pod-security.kubernetes.io/enforce=restricted --overwrite=true
+      kubectl label ns kube-system scrape=enabled --overwrite=true
+      kubectl label ns kube-system pod-security.kubernetes.io/enforce=restricted --overwrite=true
   - name: login to ecr via docker
     uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
     with:
@@ -60,7 +61,6 @@ runs:
       helm upgrade --install karpenter oci://${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com/karpenter/snapshot/karpenter \
         -n kube-system \
         --version "v0-$(git rev-parse HEAD)" \
-        --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${{ inputs.account_id }}:role/karpenter-irsa-${{ inputs.cluster_name }}" \
         --set webhook.enabled=${WEBHOOK_ENABLED} \
         --set settings.clusterName="${{ inputs.cluster_name }}" \
         --set settings.interruptionQueue="${{ inputs.cluster_name }}" \

.github/actions/e2e/install-prometheus/action.yaml

@@ -44,7 +44,6 @@ runs:
           -f ./.github/actions/e2e/install-prometheus/values.yaml \
           --set prometheus.prometheusSpec.remoteWrite[0].url=https://aps-workspaces.${{ inputs.region }}.amazonaws.com/workspaces/${{ inputs.workspace_id }}/api/v1/remote_write \
           --set prometheus.prometheusSpec.remoteWrite[0].sigv4.region=${{ inputs.region }} \
-          --set prometheus.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${{ inputs.account_id }}:role/prometheus-irsa-${{ inputs.cluster_name }}" \
           --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \
           --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \
           --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \

.github/actions/e2e/setup-cluster/action.yaml

@@ -137,6 +137,9 @@ runs:
           ebsCSIController: true
       - name: eks-pod-identity-agent
         permissionsBoundary: "arn:aws:iam::${{ inputs.account_id }}:policy/GithubActionsPermissionsBoundary"
+        configurationValues: |
+          tolerations:
+            - operator: Exists
       EOF
 
       if [[ ${{ inputs.private_cluster }} == 'true' ]]; then

test/cloudformation/iam_cloudformation.yaml

@@ -325,6 +325,7 @@ Resources:
               - sqs:ReceiveMessage
               - pricing:GetProducts
               - eks:DescribeCluster
+              - eks-auth:AssumeRoleForPodIdentity
             Resource: "*"
           - Effect: Allow
             Action: iam:PassRole