Upgade workflow

.ci/scripts/get-released-version.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Input: version to calculate previous version
+VERSION="$1"
+
+# Extract the major and minor versions
+MAJOR_VERSION=$(echo "$VERSION" | cut -d'.' -f1)
+MINOR_VERSION=$(echo "$VERSION" | cut -d'.' -f2)
+
+# Calculate the previous version (assuming it's always X.(Y-1))
+PREVIOUS_VERSION="$MAJOR_VERSION.$((MINOR_VERSION - 1))"
+
+URL="https://snapshots.elastic.co/latest/$PREVIOUS_VERSION.json"
+
+# Use curl to fetch the JSON data
+JSON_RESPONSE=$(curl -s "$URL")
+
+# Get latest snapshot version
+SNAPSHOT_VERSION=$(echo "$JSON_RESPONSE" | jq -r '.version')
+
+# Check if SNAPSHOT_VERSION is empty
+if [ -z "$SNAPSHOT_VERSION" ]; then
+    # Log an error message with variable values
+    echo "Error: The release version corresponding to $PREVIOUS_VERSION could not be found." >&2
+    exit 1
+fi
+
+# Split the version into major, minor, and patch parts
+IFS='.-' read -ra PARTS <<<"$SNAPSHOT_VERSION"
+MAJOR="${PARTS[0]}"
+MINOR="${PARTS[1]}"
+PATCH="${PARTS[2]}"
+
+# Decrement the patch version by 1
+PATCH=$((PATCH - 1))
+
+# Format the previous version
+PREVIOUS_VERSION="$MAJOR.$MINOR.$PATCH"
+
+# Output the previous version
+echo "$PREVIOUS_VERSION"

.github/workflows/test-environment.yml

@@ -19,13 +19,16 @@ on:
         required: true
         description: "Stack version: For released/BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
         default: "8.10.0"
+        type: string
       ess-region:
         required: true
         description: "Elastic Cloud deployment region"
         default: "gcp-us-west2"
+        type: string
       docker-image-override:
         required: false
         description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
+        type: string
       run-sanity-tests:
           description: "Run sanity tests after provision"
           default: false
@@ -38,6 +41,45 @@ on:
         type: string
         description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
         required: false
+  workflow_call:
+    inputs:
+      deployment_name:
+        description: Name of the deployment to create
+        type: string
+        required: true
+      elk-stack-version:
+        required: true
+        description: "Stack version: For released/BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
+        default: "8.10.0"
+        type: string
+      ess-region:
+        required: true
+        description: "Elastic Cloud deployment region"
+        default: "gcp-us-west2"
+        type: string
+      docker-image-override:
+        required: false
+        description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
+        type: string
+      run-sanity-tests:
+        description: "Run sanity tests after provision"
+        default: false
+        type: boolean
+      cleanup-env:
+        description: "Cleanup resources after provision"
+        default: false
+        type: boolean
+      ec-api-key:
+        type: string
+        description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
+        required: false
+    outputs:
+      s3-bucket:
+        description: "Terraform state s3 bucket folder"
+        value: ${{ jobs.Deploy.outputs.deploy-s3-bucket }}
+      cnvm-stack-name:
+        description: "AWS CNVM integration stack name"
+        value: ${{ jobs.Deploy.outputs.aws-cnvm-stack-name }}
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -57,18 +99,21 @@ jobs:
       run:
         working-directory: ${{ env.WORKING_DIR }}
     env:
-      TF_VAR_serverless_mode: ${{ github.event.inputs.serverless_mode }}
-      TF_VAR_stack_version: ${{ github.event.inputs.elk-stack-version }}
-      TF_VAR_ess_region: ${{ github.event.inputs.ess-region }}
-      DEPLOYMENT_NAME: ${{ github.event.inputs.deployment_name }}
+      TF_VAR_stack_version: ${{ inputs.elk-stack-version }}
+      TF_VAR_ess_region: ${{ inputs.ess-region }}
+      DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
+      TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
       S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
-      DOCKER_IMAGE_OVERRIDE: ${{ github.event.inputs.docker-image-override }}
-      STACK_VERSION: ${{ github.event.inputs.elk-stack-version }}
-      CNVM_STACK_NAME: "${{ github.event.inputs.deployment_name }}-cnvm-sanity-test-stack"
+      DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
+      STACK_VERSION: ${{ inputs.elk-stack-version }}
+      CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
     # Add "id-token" with the intended permissions.
     permissions:
       contents: 'read'
       id-token: 'write'
+    outputs:
+      deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }}
+      aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -79,7 +124,7 @@ jobs:
 
       - name: Check Deployment Name
         run: |
-          deployment_name="${{ github.event.inputs.deployment_name }}"
+          deployment_name="${{ inputs.deployment_name }}"
 
           # Check length
           if [ ${#deployment_name} -gt 20 ]; then
@@ -94,7 +139,7 @@ jobs:
           fi
 
       - name: Mask Sensitive Data
-        if: github.event.inputs.ec-api-key != ''
+        if: inputs.ec-api-key != ''
         run: |
           ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
           echo "::add-mask::$ec_api_key"
@@ -184,13 +229,16 @@ jobs:
           echo "CSPM_PUBLIC_IP=$CSPM_PUBLIC_IP" >> $GITHUB_ENV
 
       - name: Upload tf state
+        id: upload-state
         if: always()
         env:
           S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
         run: |
           aws s3 cp "./terraform.tfstate" "${{ env.S3_BUCKET }}/terraform.tfstate"
           aws s3 cp "${{ env.EC2_CSPM_KEY }}" "${{ env.S3_BUCKET }}/cspm.pem"
           aws s3 cp "${{ env.EC2_KSPM_KEY }}" "${{ env.S3_BUCKET }}/kspm.pem"
+          echo "s3-bucket-folder=${{ env.S3_BUCKET }}" >> $GITHUB_OUTPUT
+          echo "aws-cnvm-stack=${{ env.CNVM_STACK_NAME }}" >> $GITHUB_OUTPUT
 
       - name: Summary
         if: success()
@@ -320,6 +368,7 @@ jobs:
           aws s3 cp "${{ env.FLEET_API_DIR}}/kspm_d4c.yaml" "${{ env.S3_BUCKET }}/kspm_d4c.yaml"
           aws s3 cp "${{ env.FLEET_API_DIR}}/kspm_eks.yaml" "${{ env.S3_BUCKET }}/kspm_eks.yaml"
           aws s3 cp "${{ env.FLEET_API_DIR}}/cspm-linux.sh" "${{ env.S3_BUCKET }}/cspm-linux.sh"
+          aws s3 cp "${{ env.FLEET_API_DIR}}/state_data.json" "${{ env.S3_BUCKET }}/state_data.json"
 
       - name: Wait for agents to enroll
         id: wait-for-agents
@@ -328,13 +377,13 @@ jobs:
           poetry run python src/agents_enrolled.py
 
       - name: Run Sanity checks
-        if: ${{ success() && github.event.inputs.run-sanity-tests == 'true' }}
+        if: ${{ success() && inputs.run-sanity-tests == true }}
         working-directory: ./tests
         run: |
           poetry install
           poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
 
       - name: Cleanup Environment
-        if: github.event.inputs.cleanup-env == 'true'
+        if: inputs.cleanup-env == 'true'
         run: |
           just delete-cloud-env ${{ env.DEPLOYMENT_NAME }} '' "false"

.github/workflows/upgrade-environment.yml

@@ -0,0 +1,205 @@
+name: Test Upgrade Environment
+run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }}
+
+on:
+  # Ability to execute on demand
+  workflow_dispatch:
+    inputs:
+      deployment_name:
+        type: string
+        description: |
+          Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
+        required: true
+      target-elk-stack-version:
+        required: true
+        description: "Target version of the ELK stack: For BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
+        default: "8.11.0"
+        type: string
+      docker-image-override:
+        required: false
+        description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: "eu-west-1"
+  WORKING_DIR: deploy/test-environments
+  FLEET_API_DIR: fleet_api/src
+  TF_VAR_stack_version: ${{ inputs.target-elk-stack-version }}
+  TF_VAR_ess_region: gcp-us-west2
+  TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
+  DOCKER_IMAGE: ${{ inputs.docker-image-override }}
+
+jobs:
+  init:
+    runs-on: ubuntu-20.04
+    outputs:
+      stack-version: ${{ steps.set-previous-version.outputs.PREVIOUS_VERSION }}
+      ess-region: ${{ env.TF_VAR_ess_region }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Set Previous Version
+        id: set-previous-version
+        run: |
+          VERSION="${{ inputs.target-elk-stack-version }}"
+          PREVIOUS_VERSION=$(./.ci/scripts/get-released-version.sh "$VERSION")
+          echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> $GITHUB_OUTPUT
+  deploy:
+    uses: ./.github/workflows/test-environment.yml
+    needs: init
+    with:
+      deployment_name: ${{ inputs.deployment_name }}
+      elk-stack-version: ${{ needs.init.outputs.stack-version }}
+      ess-region: ${{ needs.init.outputs.ess-region }}
+      run-sanity-tests: false # Set to true once the issue at https://github.com/elastic/kibana/pull/171200 is resolved.
+    secrets: inherit
+  upgrade:
+    runs-on: ubuntu-20.04
+    needs: [init, deploy]
+    timeout-minutes: 120
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+        working-directory: ./
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry --version
+
+      - name: Install Fleet API dependencies
+        id: fleet-api-deps
+        working-directory: ${{ env.WORKING_DIR }}/fleet_api
+        run: |
+          poetry install
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::704479110758:role/Developer_eks
+          role-session-name: github-ci
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Download tf state data
+        env:
+          S3_BUCKET: ${{ needs.deploy.outputs.s3-bucket }}
+        run: |
+          aws s3 cp "${{ env.S3_BUCKET }}/terraform.tfstate" "./terraform.tfstate"
+
+      - name: Terraform Init
+        run: terraform init
+
+      - name: Terraform Validate
+        run: terraform validate
+
+      - name: Update ELK stack version
+        id: apply
+        if: success()
+        run: |
+          terraform apply --auto-approve -var="deployment_name=${{ inputs.deployment_name }}" -var="region=${{ env.AWS_REGION }}"
+
+      - name: Set Environment Output
+        id: env-output
+        run: |
+          echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV
+          echo "ES_URL=$(terraform output -raw elasticsearch_url)" >> $GITHUB_ENV
+          echo "ES_USER=$(terraform output -raw elasticsearch_username)" >> $GITHUB_ENV
+
+          export ES_PASSWORD=$(terraform output -raw elasticsearch_password)
+          echo "::add-mask::$ES_PASSWORD"
+          echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV
+
+          export EC2_CSPM=$(terraform output -raw ec2_cspm_ssh_cmd)
+          echo "::add-mask::$EC2_CSPM"
+          echo "EC2_CSPM=$EC2_CSPM" >> $GITHUB_ENV
+
+          export EC2_KSPM=$(terraform output -raw ec2_kspm_ssh_cmd)
+          echo "::add-mask::$EC2_KSPM"
+          echo "EC2_KSPM=$EC2_KSPM" >> $GITHUB_ENV
+
+          export EC2_CSPM_KEY=$(terraform output -raw ec2_cspm_key)
+          echo "::add-mask::$EC2_CSPM_KEY"
+          echo "EC2_CSPM_KEY=$EC2_CSPM_KEY" >> $GITHUB_ENV
+
+          export EC2_KSPM_KEY=$(terraform output -raw ec2_kspm_key)
+          echo "::add-mask::$EC2_KSPM_KEY"
+          echo "EC2_KSPM_KEY=$EC2_KSPM_KEY" >> $GITHUB_ENV
+
+          export KSPM_PUBLIC_IP=$(terraform output -raw ec2_kspm_public_ip)
+          echo "::add-mask::$KSPM_PUBLIC_IP"
+          echo "KSPM_PUBLIC_IP=$KSPM_PUBLIC_IP" >> $GITHUB_ENV
+
+          export CSPM_PUBLIC_IP=$(terraform output -raw ec2_cspm_public_ip)
+          echo "::add-mask::$CSPM_PUBLIC_IP"
+          echo "CSPM_PUBLIC_IP=$CSPM_PUBLIC_IP" >> $GITHUB_ENV
+
+      - name: Run Sanity checks
+        if: success()
+        working-directory: ./tests
+        env:
+          USE_K8S: false
+          STACK_VERSION: ${{ needs.init.outputs.stack-version }}
+        run: |
+          poetry install
+          poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
+
+      - name: Set Docker Image version
+        if: ${{ ! inputs.docker-image-override }}
+        env:
+          VERSION: 'docker.elastic.co/beats/elastic-agent:${{ inputs.target-elk-stack-version }}'
+        run: |
+          echo "DOCKER_IMAGE=${{ env.VERSION }}" >> $GITHUB_ENV
+
+      - name: Download Integrations data
+        env:
+          S3_BUCKET: ${{ needs.deploy.outputs.s3-bucket }}
+        run: |
+          aws s3 cp "${{ env.S3_BUCKET }}/kspm.pem" "${{ env.EC2_KSPM_KEY }}"
+          aws s3 cp "${{ env.S3_BUCKET }}/state_data.json" "${{ env.FLEET_API_DIR }}/state_data.json"
+
+      - name: Upgrade KSPM Unmanaged agent
+        run: |
+          chmod 600 ${{ env.EC2_KSPM_KEY }}
+          # Update image
+          ssh -o StrictHostKeyChecking=no -v -i ${{ env.EC2_KSPM_KEY }} "ubuntu@${{ env.KSPM_PUBLIC_IP }}" "kubectl set image daemonset elastic-agent -n kube-system elastic-agent=${{ env.DOCKER_IMAGE }}"
+
+      - name: Upgrade KSPM EKS agent
+        run: |
+          aws eks --region ${{ env.AWS_REGION }} update-kubeconfig \
+              --name $(terraform output -raw deployment_name) --alias eks-config
+          kubectl config use-context eks-config
+          kubectl set image daemonset elastic-agent -n kube-system elastic-agent=${{ env.DOCKER_IMAGE }}
+
+      - name: Upgrade Linux agents
+        working-directory: ${{ env.WORKING_DIR }}/${{ env.FLEET_API_DIR }}
+        env:
+          CNVM_STACK_NAME: ${{ needs.deploy.outputs.cnvm-stack-name }}
+          STACK_VERSION: ${{ inputs.target-elk-stack-version }}
+        run: |
+          poetry run python upgrade_agents.py
+
+      - name: Run Upgrade Sanity checks
+        if: success()
+        working-directory: ./tests
+        env:
+          STACK_VERSION: ${{ inputs.target-elk-stack-version }}
+          USE_K8S: false
+        run: |
+          poetry install
+          poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4